kubectl apply -f nodeport.yaml

master $ kubectl get svc

NAME                   TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)        AGE

kubernetes             ClusterIP   10.96.0.1               443/TCP        36m

webapp1-nodeport-svc   NodePort    10.103.188.73           80:30080/TCP   3m

master $ kubectl get pod -o wide

NAME                                           READY     STATUS    RESTARTS   AGE       IPNODE      NOMINATED NODE

webapp1-nodeport-deployment-785989576b-cjc5b   1/1       Running   0          2m        10.32.0.3

webapp1-nodeport-deployment-785989576b-tpfqr   1/1       Running   0          2m        10.32.0.5

master $ netstat -lnp|grep 30080

tcp6       0      0 :::30080                :::*                    LISTEN      7427/kube-proxy

iptables-save > iptables-dump

 # Generated by iptables-save v1.6.0 on Thu Mar 28 07:33:57 2019

*nat

 # Nodeport规则链

:KUBE-NODEPORTS - [0:0]

 # Service规则链

:KUBE-SERVICES - [0:0]

 # Nodeport和Service共用的规则链

:KUBE-SVC-J2DWGRZTH4C2LPA4 - [0:0]

:KUBE-SEP-4CGFRVESQ3AECDE7 - [0:0]

:KUBE-SEP-YLXG4RMKAICGY2B3 - [0:0]



 # 将host上30080端口的外部tcp流量转到KUBE-SVC-J2DWGRZTH4C2LPA4链

-A KUBE-NODEPORTS -p tcp -m comment --comment "default/webapp1-nodeport-svc:" -m tcp --dport 30080 -j KUBE-SVC-J2DWGRZTH4C2LPA4



 #将发送到Cluster IP 10.103.188.73的内部流量转到KUBE-SVC-J2DWGRZTH4C2LPA4链

KUBE-SERVICES -d 10.103.188.73/32 -p tcp -m comment --comment "default/webapp1-nodeport-svc: cluster IP" -m tcp --dport 80 -j KUBE-SVC-J2DWGRZTH4C2LPA4



 #将发送到webapp1-nodeport-svc的流量转交到第一个Pod（10.32.0.3）相关的规则链上，比例为50%

-A KUBE-SVC-J2DWGRZTH4C2LPA4 -m comment --comment "default/webapp1-nodeport-svc:" -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-YLXG4RMKAICGY2B3

 #将发送到webapp1-nodeport-svc的流量转交到第二个Pod（10.32.0.5）相关的规则链上

-A KUBE-SVC-J2DWGRZTH4C2LPA4 -m comment --comment "default/webapp1-nodeport-svc:" -j KUBE-SEP-4CGFRVESQ3AECDE7



 #将请求重定向到Pod 10.32.0.3

-A KUBE-SEP-YLXG4RMKAICGY2B3 -p tcp -m comment --comment "default/webapp1-nodeport-svc:" -m tcp -j DNAT --to-destination 10.32.0.3:80

 #将请求重定向到Pod 10.32.0.5

-A KUBE-SEP-4CGFRVESQ3AECDE7 -p tcp -m comment --comment "default/webapp1-nodeport-svc:" -m tcp -j DNAT --to-destination 10.32.0.5:80

备注：除采用iptables进行流量转发，NodePort应该也可以提供userspace模式以及ipvs模式，这里未就这两种模式进行实验验证。

备注：LoadBalancer类型需要云服务提供商的支持，Service中的定义只是在Kubernetes配置文件中提出了一个要求，即为该Service创建Load Balancer，至于如何创建则是由Google Cloud或Amazon Cloud等云服务商提供的，创建的Load Balancer的过程不在Kubernetes Cluster的管理范围中。

1. 外部请求先通过四层Load Balancer进入内部网络
2. Load Balancer将流量分发到后端多个主机节点上的NodePort（userspace转发）
3. 请求从NodePort进入到Ingress Controller（iptabes规则，Ingress Controller本身是一个NodePort类型的Service）
4. Ingress Controller根据Ingress rule进行七层分发，根据HTTP的URL和Host将请求分发给不同的Service（userspace转发）
5. Service将请求最终导入到后端提供服务的Pod中（iptabes规则）

func init() {

   formatter := runtime.Formatter{ChildFormatter: &log.JSONFormatter{}}

   formatter.Line = true

   log.SetFormatter(&formatter)

   log.SetOutput(os.Stdout)

   level, err := log.ParseLevel(getEnv("LOG_LEVEL", "info"))

   if err != nil {

      log.Error(err)

   }

   log.SetLevel(level)

} 

LoggerModule.forRoot({

  level: !environment.production ? NgxLoggerLevel.DEBUG : NgxLoggerLevel.INFO,

  serverLogLevel: NgxLoggerLevel.INFO

})

# Grafana

kubectl port-forward -n istio-system \

  $(kubectl get pod -n istio-system -l app=grafana \

  -o jsonpath='{.items[0].metadata.name}') 3000:3000 &

  

 # Prometheus

kubectl -n istio-system port-forward \

  $(kubectl -n istio-system get pod -l app=prometheus \

  -o jsonpath='{.items[0].metadata.name}') 9090:9090 &

  

 # Jaeger

kubectl port-forward -n istio-system \

$(kubectl get pod -n istio-system -l app=jaeger \

-o jsonpath='{.items[0].metadata.name}') 16686:16686 &

  

 # Kiali

kubectl -n istio-system port-forward \

  $(kubectl -n istio-system get pod -l app=kiali \

  -o jsonpath='{.items[0].metadata.name}') 20001:20001 &

up{namespace="dev",pod_name=~"service-.*"}

container_memory_max_usage_bytes{namespace="dev",container_name=~"service-.*"}

container_memory_max_usage_bytes{namespace="dev",container_name="service-f"}

container_network_transmit_packets_total{namespace="dev",pod_name=~"service-e-.*"}

istio_requests_total{destination_service_namespace="dev",connection_security_policy="mutual_tls",destination_app="service-a"}

istio_response_bytes_count{destination_service_namespace="dev",connection_security_policy="mutual_tls",source_app="service-a"} 

• `x-request-id`
• `x-b3-traceid`
• `x-b3-spanid`
• `x-b3-parentspanid`
• `x-b3-sampled`
• `x-b3-flags`
• `x-ot-span-context`

headers := []string{

  "x-request-id",

  "x-b3-traceid",

  "x-b3-spanid",

  "x-b3-parentspanid",

  "x-b3-sampled",

  "x-b3-flags",

  "x-ot-span-context",

}

for _, header := range headers {

  if r.Header.Get(header) != "" {

    req.Header.Add(header, r.Header.Get(header))

  }

} 

git clone --branch master --single-branch --depth 1 --no-tags \

https://github.com/garystafford/k8s-istio-observe-backend.git

1. 创建MongoDB Atlas数据库集群
2. 创建CloudAMQP RabbitMQ集群
3. 根据自己的环境调整Kubernetes的资源文件和脚本
4. 在GCP上创建GKE集群
5. 使用Helm在GKE集群中部署Istio 1.1.0版本
6. 为平台中需要暴露访问的资源创建DNS记录
7. 部署所有Go微服务，Angular UI和与GKE相关联的资源
8. 测试或为你的平台排障
9. 在第二篇中查看观察的结果

apiVersion: networking.istio.io/v1alpha3

kind: ServiceEntry

metadata:

  name: mongodb-atlas-external-mesh

spec:

  hosts:

  - {{ your_host_goes_here }}

    ports:

  - name: mongo

    number: 27017

    protocol: MONGO

    location: MESH_EXTERNAL

    resolution: NONE

apiVersion: networking.istio.io/v1alpha3

kind: ServiceEntry

metadata:

  name: cloudamqp-external-mesh

spec:

  hosts:

  - {{ your_host_goes_here }}

    ports:

  - name: rabbitmq

    number: 5672

    protocol: TCP

    location: MESH_EXTERNAL

    resolution: NONE

apiVersion: networking.istio.io/v1alpha3

kind: Gateway

metadata:

  name: demo-gateway

spec:

  selector:

    istio: ingressgateway

  servers:

  - port:

      number: 80

      name: http

      protocol: HTTP

    hosts:

    - ui.dev.example-api.com

    - ui.test.example-api.com

    - api.dev.example-api.com

    - api.test.example-api.com

apiVersion: networking.istio.io/v1alpha3

kind: VirtualService

metadata:

  name: angular-ui-dev

spec:

  hosts:

  - ui.dev.example-api.com

    gateways:

  - demo-gateway

    http:

  - match:

    - uri:

        prefix: /

    route:

    - destination:

        port:

          number: 80

        host: angular-ui.dev.svc.cluster.local

apiVersion: v1

kind: Secret

metadata:

  name: go-srv-config

type: Opaque

data:

  mongodb.conn: {{ your_base64_encoded_secret }}

  rabbitmq.conn: {{ your_base64_encoded_secret }} 

> echo -n "mongodb+srv://username:password@atlas-cluster.gcp.mongodb.net/test?retryWrites=true" | base64

bW9uZ29kYitzcnY6Ly91c2VybmFtZTpwYXNzd29yZEBhdGxhcy1jbHVzdGVyLmdjcC5tb25nb2RiLm5ldC90ZXN0P3JldHJ5V3JpdGVzPXRydWU=



[quote] echo -n "amqp://username:password@rmq.cloudamqp.com/cluster" | base64

YW1xcDovL3VzZXJuYW1lOnBhc3N3b3JkQHJtcS5jbG91ZGFtcXAuY29tL2NsdXN0ZXI=

# 对应修改这些常量

readonly PROJECT='{{ your_gcp_project_goes_here }}'

readonly CLUSTER='go-srv-demo-cluster'

readonly REGION='us-central1'

readonly MASTER_AUTH_NETS='72.231.208.0/24'

readonly GKE_VERSION='1.12.5-gke.5'

readonly MACHINE_TYPE='n1-standard-2'

readonly ISTIO_HOME='/Applications/istio-1.1.0'

helm install ${ISTIO_HOME}/install/kubernetes/helm/istio-init \

--name istio-init \

--namespace istio-system[/quote]



helm install ${ISTIO_HOME}/install/kubernetes/helm/istio \

--name istio \

--namespace istio-system \

--set prometheus.enabled=true \

--set grafana.enabled=true \

--set kiali.enabled=true \

--set tracing.enabled=true



kubectl apply --namespace istio-system \

-f ./resources/secrets/kiali.yaml

1. 四个DNS记录中存在错误，它们不是解析到负载均衡器的前端IP地址上
2. 没有为四个Istio的VirtualService资源配置正确的子域名
3. 基于Go的微服务无法访问外部MongoDB Atlas和CloudAMQP RabbiqMQ集群。可能是Kuberetes Secret资源结构不对或者是两个ServiceEntry中包含了那些到外部集群的错误的主机信息

apiVersion: v1

kind: Service

metadata:

  name: nginx

spec:

  ports:

  - port: 80

    targetPort: 80

    name: http

  selector:

    app: nginx

--serviceCluster ${POD_NAMESPACE}.$(echo ${POD_NAME} | sed -e 's/-[a-z0-9][i]-[a-z0-9][/i]$//g')

apiVersion: v1

kind: ConfigMap

metadata:

    name: istio

    namespace: istio-system

    labels:

       app: istio

       service: istio

data:

    mesh: |-

 # disable tracing mechanism for now

       enableTracing: false

 # do not specify mixer endpoints, so that sidecar containers do not send the information

    #mixerCheckServer: istio-policy.istio-system:15004

    #mixerReportServer: istio-telemetry.istio-system:15004

 # interval for envoy to check Pilot

       rdsRefreshDelay: 5s

 # default config for envoy sidecar

       defaultConfig:

      # like rdsRefreshDelay

           discoveryRefreshDelay: 5s

 # path to envoy executable

           configPath: "/etc/istio/proxy"

           binaryPath: "/usr/local/bin/envoy"

 # default name for sidecar container

           serviceCluster: istio-proxy

 # time for envoy to wait before it shuts down all existing connections

           drainDuration: 45s

           parentShutdownDuration: 1m0s

 # by default, REDIRECT rule for iptables is used. TPROXY can be used as well.

      #interceptionMode: REDIRECT

 # port for sidecar container admin panel

           proxyAdminPort: 15000

 # address for sending traces using zipkin protocol (not used as turned off in enableTracing option)

           zipkinAddress: tracing-collector.tracing:9411

 # statsd address for envoy containers metrics

      # statsdUdpAddress: aggregator:8126

 # turn off Mutual TLS

          controlPlaneAuthPolicy: NONE

 # istio-pilot listen port to report service discovery information to sidecars

          discoveryAddress: istio-pilot.istio-system:15007

initContainers:

   - name: istio-init

       args:

       - -p

       - "15001"

       - -u

       - "1337"

       - -m

       - REDIRECT

       - -i

       - '*'

       - -b

       - '*'

       - -d

       - ""

       image: istio/proxy_init:1.0.0

       imagePullPolicy: IfNotPresent

       resources:

          limits:

            memory: 128Mi

       securityContext:

          capabilities:

              add:

              - NET_ADMIN

       - name: istio-proxy

         command:

         - "bash"

         - "-c"

         - |

           exec /usr/local/bin/pilot-agent proxy sidecar \

           --configPath \

           /etc/istio/proxy \

           --binaryPath \

           /usr/local/bin/envoy \

           --serviceCluster \

           service-name \

           --drainDuration \

           45s \

           --parentShutdownDuration \

           1m0s \

           --discoveryAddress \

           istio-pilot.istio-system:15007 \

           --discoveryRefreshDelay \

           1s \

           --connectTimeout \

           10s \

           --proxyAdminPort \

           "15000" \

           --controlPlaneAuthPolicy \

           NONE

         env:

         - name: POD_NAME

           valueFrom:

             fieldRef:

               fieldPath: metadata.name

         - name: POD_NAMESPACE

           valueFrom:

             fieldRef:

               fieldPath: metadata.namespace

         - name: INSTANCE_IP

           valueFrom:

             fieldRef:

               fieldPath: status.podIP

         - name: ISTIO_META_POD_NAME

           valueFrom:

             fieldRef:

               fieldPath: metadata.name

         - name: ISTIO_META_INTERCEPTION_MODE

           value: REDIRECT

         image: istio/proxyv2:1.0.0

         imagePullPolicy: IfNotPresent

         resources:

           requests:

             cpu: 100m

             memory: 128Mi

           limits:

             memory: 2048Mi

         securityContext:

           privileged: false

           readOnlyRootFilesystem: true

           runAsUser: 1337

         volumeMounts:

         - mountPath: /etc/istio/proxy

           name: istio-envoy

route:

    - destination:

        host: productpage

        port:

          number: 9080

spec:

  host: productpage

  subsets:

  - labels:

      version: v1

    name: v1

  trafficPolicy:

    connectionPool:

      http:

        http1MaxPendingRequests: 1

        maxRequestsPerConnection: 1

      tcp:

        maxConnections: 1

    tls:

apiVersion: networking.istio.io/v1alpha3

kind: ServiceEntry

metadata:

  name: external-svc-mongocluster

spec:

  hosts:

  - mymongodb.somedomain # not used

  addresses:

  - 192.192.192.192/24 # VIPs

  ports:

  - number: 27018

    name: mongodb

    protocol: MONGO

  location: MESH_INTERNAL

  resolution: STATIC

  endpoints:

  - address: 2.2.2.2

  - address: 3.3.3.3

spec:

  selector:

    istio: ingressgateway

  servers:

  - hosts:

    - '*'

    port:

      name: http

      number: 80

      protocol: HTTP

Type:

paperback

Pages:

200

Publisher:

PublisherA

Language:

English

ISBN-10:

1234567890

ISBN-13:

123-1234567890

An extremely entertaining play by Shakespeare. The slapstick humour is refreshing!

Absolutely fun and entertaining. The play lacks thematic depth when compared to other plays by Shakespeare.

def getProducts():

        return [

            {

                'id': 0,

                'title': 'The Comedy of Errors',

                'descriptionHtml': 'Wikipedia Summary: The Comedy of Errors is one of < b>William Shakespeare\'s< /b> early plays. It is his shortest and one of his most farcical comedies, with a major part of the humour coming from slapstick and mistaken identity, in addition to puns and word play.'

            } 

details = {

"name" : "http://details{0}:9080".format(servicesDomain),

"endpoint" : "details",

"children" : []

}





ratings = {

"name" : "http://ratings{0}:9080".format(servicesDomain),

"endpoint" : "ratings",

"children" : []

}





reviews = {

"name" : "http://reviews{0}:9080".format(servicesDomain),

"endpoint" : "reviews",

"children" : [ratings]

}





productpage = {

"name" : "http://details{0}:9080".format(servicesDomain),

"endpoint" : "details",

"children" : [details, reviews]

}





service_dict = {

"productpage" : productpage,

"details" : details,

"reviews" : reviews,

} 

private String getJsonResponse (String productId, int starsReviewer1, int starsReviewer2) {

    String result = "{";

    result += "\"id\": \"" + productId + "\",";

    result += "\"reviews\": [";

 

    // reviewer 1:

    result += "{";

    result += "  \"reviewer\": \"Reviewer1\",";

    result += "  \"text\": \"An extremely entertaining play by Shakespeare. The slapstick humour is refreshing!\"";

  if (ratings_enabled) {

    if (starsReviewer1 != -1) {

      result += ", \"rating\": {\"stars\": " + starsReviewer1 + ", \"color\": \"" + star_color + "\"}";

    }

    else {

      result += ", \"rating\": {\"error\": \"Ratings service is currently unavailable\"}";

    }

  }

    result += "},";

     

    // reviewer 2:

    result += "{";

    result += "  \"reviewer\": \"Reviewer2\",";

    result += "  \"text\": \"Absolutely fun and entertaining. The play lacks thematic depth when compared to other plays by Shakespeare.\"";

  if (ratings_enabled) {

    if (starsReviewer2 != -1) {

      result += ", \"rating\": {\"stars\": " + starsReviewer2 + ", \"color\": \"" + star_color + "\"}";

    }

    else {

      result += ", \"rating\": {\"error\": \"Ratings service is currently unavailable\"}";

    }

  }

    result += "}";

     

    result += "]";

    result += "}";

 

    return result;

} 

private final static String star_color = System.getenv("STAR_COLOR") == null ? "black" : System.getenv("STAR_COLOR");

#java build the app.

docker run --rm -v "$(pwd)":/home/gradle/project -w /home/gradle/project gradle:4.8.1 gradle clean build

pushd reviews-wlpcfg

  #plain build -- no ratings

  docker build -t "istio/examples-bookinfo-reviews-v1:${VERSION}" -t istio/examples-bookinfo-reviews-v1:latest --build-arg service_version=v1 .

  #with ratings black stars

  docker build -t "istio/examples-bookinfo-reviews-v2:${VERSION}" -t istio/examples-bookinfo-reviews-v2:latest --build-arg service_version=v2 \

     --build-arg enable_ratings=true .

  #with ratings red stars

  docker build -t "istio/examples-bookinfo-reviews-v3:${VERSION}" -t istio/examples-bookinfo-reviews-v3:latest --build-arg service_version=v3 \

     --build-arg enable_ratings=true --build-arg star_color=red .

#!/bin/sh

    set -e

    mongoimport --host localhost --db test --collection ratings --drop --file /app/data/ratings_data.json

{rating: 5}

{rating: 4} 

# Initialize a mysql db with a 'test' db and be able test productpage with it.

 # mysql -h 127.0.0.1 -ppassword < mysqldb-init.sql

 

CREATE DATABASE test;

USE test;

 

CREATE TABLE `ratings` (

  `ReviewID` INT NOT NULL,

  `Rating` INT,

  PRIMARY KEY (`ReviewID`)

);

INSERT INTO ratings (ReviewID, Rating) VALUES (1, 5);

INSERT INTO ratings (ReviewID, Rating) VALUES (2, 4);

* We default to using mongodb, if DB_TYPE is not set to mysql.

 */

if (process.env.SERVICE_VERSION === 'v2') {

  if (process.env.DB_TYPE === 'mysql') {

    var mysql = require('mysql')

    var hostName = process.env.MYSQL_DB_HOST

    var portNumber = process.env.MYSQL_DB_PORT

    var username = process.env.MYSQL_DB_USER

    var password = process.env.MYSQL_DB_PASSWORD

  } else {

    var MongoClient = require('mongodb').MongoClient

    var url = process.env.MONGO_DB_URL

  }

}

 

dispatcher.onGet(/^\/ratings\/[0-9]*/, function (req, res) {

  var productIdStr = req.url.split('/').pop()

  var productId = parseInt(productIdStr)

 

  if (Number.isNaN(productId)) {

    res.writeHead(400, {'Content-type': 'application/json'})

    res.end(JSON.stringify({error: 'please provide numeric product ID'}))

  } else if (process.env.SERVICE_VERSION === 'v2') {

    var firstRating = 0

    var secondRating = 0

 

    if (process.env.DB_TYPE === 'mysql') {

      var connection = mysql.createConnection({

        host: hostName,

        port: portNumber,

        user: username,

        password: password,

        database: 'test'

      })

 

      connection.connect()

      connection.query('SELECT Rating FROM ratings', function (err, results, fields) {

        if (err) {

          res.writeHead(500, {'Content-type': 'application/json'})

          res.end(JSON.stringify({error: 'could not connect to ratings database'}))

        } else {

          if (results[0]) {

            firstRating = results[0].Rating

          }

          if (results[1]) {

            secondRating = results[1].Rating

          }

          var result = {

            id: productId,

            ratings: {

              Reviewer1: firstRating,

              Reviewer2: secondRating

            }

          }

          res.writeHead(200, {'Content-type': 'application/json'})

          res.end(JSON.stringify(result))

        }

      }) 

[root@master networking]# cat virtual-service-reviews-v3.yaml

apiVersion: networking.istio.io/v1alpha3

kind: VirtualService

metadata:

  name: reviews

spec:

  hosts:

    - reviews

  http:

  - route:

    - destination:

        host: reviews

        subset: v3      version: v3

[root@master networking]# cat virtual-service-reviews-80-20.yaml

apiVersion: networking.istio.io/v1alpha3

kind: VirtualService

metadata:

  name: reviews

spec:

  hosts:

    - reviews

  http:

  - route:

    - destination:

        host: reviews

        subset: v1

      weight: 80

    - destination:

        host: reviews

        subset: v2

      weight: 20

[root@master ~]# cat speedrule.yaml

apiVersion: "config.istio.io/v1alpha2"

kind: memquota

metadata:

  name: handler

  namespace: myproject

spec:

  quotas:

  - name: requestcount.quota.myproject

    # default rate limit is 5000qps

    maxAmount: 5000

    validDuration: 1s

    # The first matching override is applied.

    # A requestcount instance is checked against override dimensions.

    overrides:

    - dimensions:

        destination: reviews

        source: productpage

        destinationVersion: v3

      maxAmount: 1

      validDuration: 1s

 

recommendation_rate_limit_handler.yml 文件声明了 requestcount quota。

[root@master ~]# cat recommendation_rate_limit_handler.yml

apiVersion: "config.istio.io/v1alpha2"

kind: quota

metadata:

  name: requestcount

  namespace: myproject

spec:

  dimensions:

    source: source.labels["app"] | source.service | "unknown"

    sourceVersion: source.labels["version"] | "unknown"

    destination: destination.labels["app"] | destination.service | "unknown"

    destinationVersion: destination.labels["version"] | "unknown"

 ---

apiVersion: "config.istio.io/v1alpha2"

kind: rule

metadata:

  name: quota

  namespace: myproject

spec:

  actions:

  - handler: handler.memquota

    instances:

    - requestcount.quota

 ---

apiVersion: config.istio.io/v1alpha2

kind: QuotaSpec

metadata:

  creationTimestamp: null

  name: request-count

  namespace: myproject

spec:

  rules:

  - quotas:

    - charge: 1

      quota: RequestCount

 ---

apiVersion: config.istio.io/v1alpha2

kind: QuotaSpecBinding

metadata:

  creationTimestamp: null

  name: request-count

  namespace: myproject

spec:

  quotaSpecs:

  - name: request-count

    namespace: myproject

  services:

  - name: productpage

    namespace: myproject

  - name: details

    namespace: myproject

  - name: reviews

    namespace: myproject

while true; do curl http://istio-ingressgateway-istio-system.apps.example.com/productpage; sleep .1; done

spec:

  host: productpage

  subsets:

  - labels:

      version: v1

    name: v1

  trafficPolicy:

    connectionPool:

      http:

        http1MaxPendingRequests: 1

        maxRequestsPerConnection: 1

      tcp:

        maxConnections: 1

    tls:

      mode: ISTIO_MUTUAL

[root@master ~]# cat acl-blacklist.yml

apiVersion: "config.istio.io/v1alpha2"

kind: denier

metadata:

  name: denycustomerhandler

spec:

  status:

    code: 7

    message: Not allowed

 ---

apiVersion: "config.istio.io/v1alpha2"

kind: checknothing

metadata:

  name: denycustomerrequests

spec:

 ---

apiVersion: "config.istio.io/v1alpha2"

kind: rule

metadata:

  name: denycustomer

spec:

  match: destination.labels["app"] == "details" && source.labels["app"]=="productpage"

  actions:

  - handler: denycustomerhandler.denier

instances: [ denycustomerrequests.checknothing ] 

wget https://github.com/istio/istio/releases/download/1.0.6/istio-1.0.6-linux.tar.gz

tar -zxvf istio-1.0.6-linux.tar.gz

cd istio-1.0.6

# tree install/kubernetes/helm/istio

......

......

31 directories, 139 files

kubectl apply -f install/kubernetes/helm/istio/templates/crds.yaml

kubectl get CustomResourceDefinition

tracing:

  enabled: true

servicegraph:

  enabled: true

kiali:

  enabled: true

grafana:

  enabled: true

# echo -n 'admin' | base64

YWRtaW4=

# echo -n '1f2d1e2e67df' | base64

MWYyZDFlMmU2N2Rm

# cat <
apiVersion: v1

kind: Secret

metadata:

  name: kiali

  namespace: istio-system

  labels:

    app: kiali

type: Opaque

data:

  username: YWRtaW4=

  passphrase: MWYyZDFlMmU2N2Rm

EOF

helm install install/kubernetes/helm/istio --name istio --namespace istio-system -f istio.yaml

# kubectl get pod -n istio-system

NAME                                      READY   STATUS      RESTARTS   AGE

grafana-59b8896965-5f9j2                  1/1     Running     0          23m

istio-citadel-6f444d9999-s9jrc            1/1     Running     0          23m

istio-egressgateway-6d79447874-ssbc4      1/1     Running     0          23m

istio-galley-685bb48846-mvf5w             1/1     Running     0          23m

istio-grafana-post-install-6m256          0/1     Completed   0          23m

istio-ingressgateway-5b64fffc9f-mrl9t     1/1     Running     0          23m

istio-pilot-8645f5655b-k6fcz              2/2     Running     0          23m

istio-policy-547d64b8d7-6dgkp             2/2     Running     0          23m

istio-sidecar-injector-5d8dd9448d-zfdsb   1/1     Running     0          23m

istio-telemetry-c5488fc49-qwwcv           2/2     Running     0          23m

istio-tracing-6b994895fd-4vjfx            1/1     Running     0          23m

kiali-5f9ffff7cf-jqk8p                    1/1     Running     0          23m

prometheus-76b7745b64-xjzmm               1/1     Running     0          23m

servicegraph-cb9b94c-mlhjm                1/1     Running     0          23m

kubectl label namespace  istio-injection=enabled

kubectl create -n  -f .yaml

kubectl create -f <(istioctl kube-inject -f .yaml)

# kubectl label namespace default istio-injection=enabled

 # kubectl apply -f samples/bookinfo/platform/kube/bookinfo.yaml

 # kubectl apply -f samples/bookinfo/networking/bookinfo-gateway.yaml

 # kubectl apply -f samples/bookinfo/platform/consul/destination-rule-all.yaml

kubectl -n istio-system get svc istio-ingressgateway -o jsonpath='{.spec.ports[0].nodePort}'

# cat istio-ingress.yaml 

 ---

apiVersion: extensions/v1beta1

kind: Ingress

metadata:

  name: jaeger-query

  namespace: istio-system

  annotations:

    kubernetes.io/ingress.class: traefik

spec:

  rules:

  - host: istio.jaeger-query.com

    http:

      paths:

      - path: /

        backend:

          serviceName: jaeger-query

          servicePort: 16686



---

apiVersion: extensions/v1beta1

kind: Ingress

metadata:

  name: prometheus

  namespace: istio-system

  annotations:

    kubernetes.io/ingress.class: traefik

spec:

  rules:

  - host: istio.prometheus.com

    http:

      paths:

      - path: /

        backend:

          serviceName: prometheus

          servicePort: 9090



---

apiVersion: extensions/v1beta1

kind: Ingress

metadata:

  name: grafana

  namespace: istio-system

  annotations:

    kubernetes.io/ingress.class: traefik

spec:

  rules:

  - host: istio.grafana.com

    http:

      paths:

      - path: /

        backend:

          serviceName: grafana

          servicePort: 3000



---

apiVersion: extensions/v1beta1

kind: Ingress

metadata:

  name: kiali

  namespace: istio-system

  annotations:

    kubernetes.io/ingress.class: traefik

spec:

  rules:

  - host: istio.kiali.com

    http:

      paths:

      - path: /

        backend:

          serviceName: kiali

          servicePort: 20001

kubectl apply -f istio-ingress.yaml

172.16.0.180 istio.prometheus.com

172.16.0.180  istio.jaeger-query.com

172.16.0.180  istio.grafana.com

172.16.0.180  istio.kiali.com

kubectl apply -f nodeport.yaml

master $ kubectl get svc

NAME                   TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)        AGE

kubernetes             ClusterIP   10.96.0.1               443/TCP        36m

webapp1-nodeport-svc   NodePort    10.103.188.73           80:30080/TCP   3m

master $ kubectl get pod -o wide

NAME                                           READY     STATUS    RESTARTS   AGE       IPNODE      NOMINATED NODE

webapp1-nodeport-deployment-785989576b-cjc5b   1/1       Running   0          2m        10.32.0.3

webapp1-nodeport-deployment-785989576b-tpfqr   1/1       Running   0          2m        10.32.0.5

master $ netstat -lnp|grep 30080

tcp6       0      0 :::30080                :::*                    LISTEN      7427/kube-proxy

iptables-save > iptables-dump

 # Generated by iptables-save v1.6.0 on Thu Mar 28 07:33:57 2019

*nat

 # Nodeport规则链

:KUBE-NODEPORTS - [0:0]

 # Service规则链

:KUBE-SERVICES - [0:0]

 # Nodeport和Service共用的规则链

:KUBE-SVC-J2DWGRZTH4C2LPA4 - [0:0]

:KUBE-SEP-4CGFRVESQ3AECDE7 - [0:0]

:KUBE-SEP-YLXG4RMKAICGY2B3 - [0:0]



 # 将host上30080端口的外部tcp流量转到KUBE-SVC-J2DWGRZTH4C2LPA4链

-A KUBE-NODEPORTS -p tcp -m comment --comment "default/webapp1-nodeport-svc:" -m tcp --dport 30080 -j KUBE-SVC-J2DWGRZTH4C2LPA4



 #将发送到Cluster IP 10.103.188.73的内部流量转到KUBE-SVC-J2DWGRZTH4C2LPA4链

KUBE-SERVICES -d 10.103.188.73/32 -p tcp -m comment --comment "default/webapp1-nodeport-svc: cluster IP" -m tcp --dport 80 -j KUBE-SVC-J2DWGRZTH4C2LPA4



 #将发送到webapp1-nodeport-svc的流量转交到第一个Pod（10.32.0.3）相关的规则链上，比例为50%

-A KUBE-SVC-J2DWGRZTH4C2LPA4 -m comment --comment "default/webapp1-nodeport-svc:" -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-YLXG4RMKAICGY2B3

 #将发送到webapp1-nodeport-svc的流量转交到第二个Pod（10.32.0.5）相关的规则链上

-A KUBE-SVC-J2DWGRZTH4C2LPA4 -m comment --comment "default/webapp1-nodeport-svc:" -j KUBE-SEP-4CGFRVESQ3AECDE7



 #将请求重定向到Pod 10.32.0.3

-A KUBE-SEP-YLXG4RMKAICGY2B3 -p tcp -m comment --comment "default/webapp1-nodeport-svc:" -m tcp -j DNAT --to-destination 10.32.0.3:80

 #将请求重定向到Pod 10.32.0.5

-A KUBE-SEP-4CGFRVESQ3AECDE7 -p tcp -m comment --comment "default/webapp1-nodeport-svc:" -m tcp -j DNAT --to-destination 10.32.0.5:80

备注：除采用iptables进行流量转发，NodePort应该也可以提供userspace模式以及ipvs模式，这里未就这两种模式进行实验验证。

备注：LoadBalancer类型需要云服务提供商的支持，Service中的定义只是在Kubernetes配置文件中提出了一个要求，即为该Service创建Load Balancer，至于如何创建则是由Google Cloud或Amazon Cloud等云服务商提供的，创建的Load Balancer的过程不在Kubernetes Cluster的管理范围中。

1. 外部请求先通过四层Load Balancer进入内部网络
2. Load Balancer将流量分发到后端多个主机节点上的NodePort（userspace转发）
3. 请求从NodePort进入到Ingress Controller（iptabes规则，Ingress Controller本身是一个NodePort类型的Service）
4. Ingress Controller根据Ingress rule进行七层分发，根据HTTP的URL和Host将请求分发给不同的Service（userspace转发）
5. Service将请求最终导入到后端提供服务的Pod中（iptabes规则）

• 流量管理

• 安全

• 安装/配置

• 支持的环境

• 监测

• 策略管理

• 性能

• 单个控制平面即可满足不同集群之间网络连接和pod之间IP寻址。

• 通过使用具有单个控制平面的集群边界网关（egress和ingress）即可访问多个集群上的kubernetes API服务器。

对于这种综合工作负载，Istio的Envoy代理使用比Linkerd多50%的CPU。Linkerd的控制平面使用了一小部分Istio，特别是在考虑“core”组件时。

在本实验中，与基线设置相比，Linkerd2-meshed设置和Istio-meshed设置都经历了更高的延迟和更低的吞吐量。在Istio-meshed设置中产生的延迟高于在Linkerd2-meshed设置中观察到的延迟。Linkerd2-Meshed设置能够处理比Istio-Meshed设置更高的HTTP和GRPC ping吞吐量。

意识到Mixer所增加的处理时间，Istio团队目前正致力于重写Mixer组件：“Mixer将用c++重新并直接嵌入到Envoy中。将不再有任何独立的Mixer服务。这将提高性能并降低运营复杂性。”

本文由博云研究院原创翻译发表，转载请注明出处。

func init() {

   formatter := runtime.Formatter{ChildFormatter: &log.JSONFormatter{}}

   formatter.Line = true

   log.SetFormatter(&formatter)

   log.SetOutput(os.Stdout)

   level, err := log.ParseLevel(getEnv("LOG_LEVEL", "info"))

   if err != nil {

      log.Error(err)

   }

   log.SetLevel(level)

} 

LoggerModule.forRoot({

  level: !environment.production ? NgxLoggerLevel.DEBUG : NgxLoggerLevel.INFO,

  serverLogLevel: NgxLoggerLevel.INFO

})

# Grafana

kubectl port-forward -n istio-system \

  $(kubectl get pod -n istio-system -l app=grafana \

  -o jsonpath='{.items[0].metadata.name}') 3000:3000 &

  

 # Prometheus

kubectl -n istio-system port-forward \

  $(kubectl -n istio-system get pod -l app=prometheus \

  -o jsonpath='{.items[0].metadata.name}') 9090:9090 &

  

 # Jaeger

kubectl port-forward -n istio-system \

$(kubectl get pod -n istio-system -l app=jaeger \

-o jsonpath='{.items[0].metadata.name}') 16686:16686 &

  

 # Kiali

kubectl -n istio-system port-forward \

  $(kubectl -n istio-system get pod -l app=kiali \

  -o jsonpath='{.items[0].metadata.name}') 20001:20001 &

up{namespace="dev",pod_name=~"service-.*"}

container_memory_max_usage_bytes{namespace="dev",container_name=~"service-.*"}

container_memory_max_usage_bytes{namespace="dev",container_name="service-f"}

container_network_transmit_packets_total{namespace="dev",pod_name=~"service-e-.*"}

istio_requests_total{destination_service_namespace="dev",connection_security_policy="mutual_tls",destination_app="service-a"}

istio_response_bytes_count{destination_service_namespace="dev",connection_security_policy="mutual_tls",source_app="service-a"} 

• `x-request-id`
• `x-b3-traceid`
• `x-b3-spanid`
• `x-b3-parentspanid`
• `x-b3-sampled`
• `x-b3-flags`
• `x-ot-span-context`

headers := []string{

  "x-request-id",

  "x-b3-traceid",

  "x-b3-spanid",

  "x-b3-parentspanid",

  "x-b3-sampled",

  "x-b3-flags",

  "x-ot-span-context",

}

for _, header := range headers {

  if r.Header.Get(header) != "" {

    req.Header.Add(header, r.Header.Get(header))

  }

} 

git clone --branch master --single-branch --depth 1 --no-tags \

https://github.com/garystafford/k8s-istio-observe-backend.git

1. 创建MongoDB Atlas数据库集群
2. 创建CloudAMQP RabbitMQ集群
3. 根据自己的环境调整Kubernetes的资源文件和脚本
4. 在GCP上创建GKE集群
5. 使用Helm在GKE集群中部署Istio 1.1.0版本
6. 为平台中需要暴露访问的资源创建DNS记录
7. 部署所有Go微服务，Angular UI和与GKE相关联的资源
8. 测试或为你的平台排障
9. 在第二篇中查看观察的结果

apiVersion: networking.istio.io/v1alpha3

kind: ServiceEntry

metadata:

  name: mongodb-atlas-external-mesh

spec:

  hosts:

  - {{ your_host_goes_here }}

    ports:

  - name: mongo

    number: 27017

    protocol: MONGO

    location: MESH_EXTERNAL

    resolution: NONE

apiVersion: networking.istio.io/v1alpha3

kind: ServiceEntry

metadata:

  name: cloudamqp-external-mesh

spec:

  hosts:

  - {{ your_host_goes_here }}

    ports:

  - name: rabbitmq

    number: 5672

    protocol: TCP

    location: MESH_EXTERNAL

    resolution: NONE

apiVersion: networking.istio.io/v1alpha3

kind: Gateway

metadata:

  name: demo-gateway

spec:

  selector:

    istio: ingressgateway

  servers:

  - port:

      number: 80

      name: http

      protocol: HTTP

    hosts:

    - ui.dev.example-api.com

    - ui.test.example-api.com

    - api.dev.example-api.com

    - api.test.example-api.com

apiVersion: networking.istio.io/v1alpha3

kind: VirtualService

metadata:

  name: angular-ui-dev

spec:

  hosts:

  - ui.dev.example-api.com

    gateways:

  - demo-gateway

    http:

  - match:

    - uri:

        prefix: /

    route:

    - destination:

        port:

          number: 80

        host: angular-ui.dev.svc.cluster.local

apiVersion: v1

kind: Secret

metadata:

  name: go-srv-config

type: Opaque

data:

  mongodb.conn: {{ your_base64_encoded_secret }}

  rabbitmq.conn: {{ your_base64_encoded_secret }} 

> echo -n "mongodb+srv://username:password@atlas-cluster.gcp.mongodb.net/test?retryWrites=true" | base64

bW9uZ29kYitzcnY6Ly91c2VybmFtZTpwYXNzd29yZEBhdGxhcy1jbHVzdGVyLmdjcC5tb25nb2RiLm5ldC90ZXN0P3JldHJ5V3JpdGVzPXRydWU=



[quote] echo -n "amqp://username:password@rmq.cloudamqp.com/cluster" | base64

YW1xcDovL3VzZXJuYW1lOnBhc3N3b3JkQHJtcS5jbG91ZGFtcXAuY29tL2NsdXN0ZXI=

# 对应修改这些常量

readonly PROJECT='{{ your_gcp_project_goes_here }}'

readonly CLUSTER='go-srv-demo-cluster'

readonly REGION='us-central1'

readonly MASTER_AUTH_NETS='72.231.208.0/24'

readonly GKE_VERSION='1.12.5-gke.5'

readonly MACHINE_TYPE='n1-standard-2'

readonly ISTIO_HOME='/Applications/istio-1.1.0'

helm install ${ISTIO_HOME}/install/kubernetes/helm/istio-init \

--name istio-init \

--namespace istio-system[/quote]



helm install ${ISTIO_HOME}/install/kubernetes/helm/istio \

--name istio \

--namespace istio-system \

--set prometheus.enabled=true \

--set grafana.enabled=true \

--set kiali.enabled=true \

--set tracing.enabled=true



kubectl apply --namespace istio-system \

-f ./resources/secrets/kiali.yaml

1. 四个DNS记录中存在错误，它们不是解析到负载均衡器的前端IP地址上
2. 没有为四个Istio的VirtualService资源配置正确的子域名
3. 基于Go的微服务无法访问外部MongoDB Atlas和CloudAMQP RabbiqMQ集群。可能是Kuberetes Secret资源结构不对或者是两个ServiceEntry中包含了那些到外部集群的错误的主机信息