feat(py): flask plugins pinpoint-apm#538

- plugins testcase
- update docker devcontainer

close pinpoint-apm#538

.devcontainer/Dockerfile

@@ -3,7 +3,7 @@ USER root
 ## install developmental env
 ENV TZ=Asia/Shanghai
 ENV LC_CTYPE=en_US.UTF-8
-ENV LC_ALL=en_US.UTF-8
+# ENV LC_ALL=en_US.UTF-8
 # referr from https://code.visualstudio.com/remote/advancedcontainers/add-nonroot-user
 ARG USERNAME=pinpoint
 ARG USER_UID=1000
@@ -28,10 +28,10 @@ RUN groupadd --gid $USER_GID $USERNAME \
 
 
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-RUN apt update && apt install -y gcc g++ gdb  pkg-config  libxml2-dev libsqlite3-dev libcurl4-openssl-dev zlib1g-dev make cmake wget autoconf git clangd python3-pip clang-format protobuf-compiler vim
-## enable php-7.4
+RUN apt update && apt install -y apt-utils gcc g++ gdb  pkg-config  libxml2-dev libsqlite3-dev libcurl4-openssl-dev zlib1g-dev make cmake wget autoconf git clangd iputils-ping python3 python3.8-venv python3-pip clang-format protobuf-compiler vim
+# enable php-7.4
 ENV PHP_VESION=php-7.4.33
-RUN cd ~ && wget https://www.php.net/distributions/${PHP_VESION}.tar.gz && tar xvf ${PHP_VESION}.tar.gz && cd ${PHP_VESION} && ./configure --prefix=/opt/${PHP_VESION} --enable-fpm   --enable-opcache   --enable-debug && make -j && make install
+RUN cd ~ && wget http://mirror.internode.on.net/pub/php/${PHP_VESION}.tar.gz -q --show-progress && tar xvf ${PHP_VESION}.tar.gz && cd ${PHP_VESION} && ./configure --prefix=/opt/${PHP_VESION} --enable-fpm   --enable-opcache   --enable-debug && make -j && make install
 ENV PATH="$PATH:/opt/php/bin/:/opt/php/sbin/"
 COPY php.ini /opt/$PHP_VESION/lib
 
@@ -66,4 +66,4 @@ RUN wget https://go.dev/dl/go1.18.10.linux-amd64.tar.gz && rm -rf /usr/local/go
 ENV PATH=$PATH:/usr/local/go/bin
 ENV GO_PATH=~/go
 ENV PATH=$PATH:/$GO_PATH/bin
-RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@latest && go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
+RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@latest && go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest

.devcontainer/devcontainer.json

@@ -1,22 +1,10 @@
 {
     "name": "PINPOINT php Dev",
-    "dockerFile": "Dockerfile",
-    "runArgs": [
-        "--user=pinpoint",
-        "--cap-add=SYS_PTRACE",
-        "--cap-add=NET_RAW",
-        "--cap-add=NET_ADMIN",
-        "--security-opt=seccomp=unconfined",
-        "--volume=${env:HOME}:${env:HOME}",
-        "--volume=/var/run/docker.sock:/var/run/docker.sock",
-        // "--volume=envoy-build:/build",
-        "--network=host",
-        // Uncomment next line if you have devcontainer.env
-        // "--env-file=.devcontainer/devcontainer.env"
+    "service": "pinpoint",
+    "dockerComposeFile": [
+        "docker-compose.yaml"
     ],
-    "containerEnv": {
-        "ENVOY_SRCDIR": "${containerWorkspaceFolder}",
-    },
+    "workspaceFolder": "/workspace",
     "settings": {
         "terminal.integrated.shell.linux": "/bin/bash",
         "bazel.buildifierFixOnFormat": true,
@@ -33,22 +21,24 @@
     },
     "remoteUser": "pinpoint",
     "containerUser": "pinpoint",
-    "postCreateCommand": ".devcontainer/setup.sh",
+    // "postCreateCommand": ".devcontainer/setup.sh",
     "extensions": [
-		"zxh404.vscode-proto3",
-		"llvm-vs-code-extensions.vscode-clangd",
-		"vadimcn.vscode-lldb",
-		"webfreak.debug",
-		"ms-python.python",
-		"xaver.clang-format",
-		"zxh404.vscode-proto3",
-		"Gruntfuggly.todo-tree",
-		"msyrus.go-doc",
-		"golang.go",
-		"bierner.github-markdown-preview",
-		"bierner.markdown-emoji",
-		"ms-vscode.cmake-tools",
-		"matepek.vscode-catch2-test-adapter",
-		"adam-bender.commit-message-editor"
-	]
+        "zxh404.vscode-proto3",
+        "llvm-vs-code-extensions.vscode-clangd",
+        "vadimcn.vscode-lldb",
+        "webfreak.debug",
+        "ms-python.python",
+        "xaver.clang-format",
+        "zxh404.vscode-proto3",
+        "Gruntfuggly.todo-tree",
+        "msyrus.go-doc",
+        "golang.go",
+        "bierner.github-markdown-preview",
+        "bierner.markdown-emoji",
+        "ms-vscode.cmake-tools",
+        "matepek.vscode-catch2-test-adapter",
+        "adam-bender.commit-message-editor",
+        "formulahendry.vscode-mysql",
+        "ms-python.autopep8"
+    ]
 }

.devcontainer/docker-compose.yaml

@@ -0,0 +1,49 @@
+version: '3.9'
+
+services:
+  dev-mysql:
+    image: mysql:5.7
+    restart: always
+    container_name: dev-mysql
+    hostname: dev-mysql
+    environment:
+      MYSQL_ROOT_PASSWORD: 'password'
+    ports:
+      - '3306:3306'      
+    expose:
+      - '3306'
+    volumes:
+      - ..:/workspace:cached
+  redis:
+    image: redis:alpine
+    command: redis-server --port 6379
+    container_name: redis_boot
+    hostname: redis_boot
+    labels:
+      - "name=redis"
+      - "mode=standalone"
+    ports:
+      - 6379:6379
+
+  pinpoint:
+    container_name: pinpoint
+    command: sleep infinity
+    hostname: pinpoint
+    build:
+      context: ./
+      dockerfile: Dockerfile
+      # args:
+    volumes:
+      - ..:/workspace:cached
+      - /var/run/docker.sock:/var/run/docker.sock
+      # - ${env:HOME}:/workspace
+    security_opt:
+      - seccomp=unconfined
+    cap_add:
+      - SYS_PTRACE
+      - NET_RAW
+      - NET_ADMIN
+    user: pinpoint
+
+
+

MANIFEST.in

@@ -1,4 +1,6 @@
 include *.txt README.md NOTICE LICENSE  MANIFEST.in
 recursive-include common *.cmake *.h *.txt *.in *.cc *.c *.cpp *.h *.inl
-recursive-include src/PY *.py *.c *.h *.txt
-prune common/build
+recursive-include src/PY *.c *.h
+recursive-include plugins/PY/pinpointPy *.py
+recursive-exclude plugins/PY/pinpointPy *_test.py
+global-exclude *test.py *.pyc Test*.py

collector-agent/Dockerfile

@@ -1,8 +1,11 @@
 FROM golang:1.18-alpine3.17 as builder
 WORKDIR /apps
-COPY  collector-agent /apps/collector-agent
-RUN  cd /apps/collector-agent && go mod tidy && go build  -o  CollectorAgent
+USER root
+RUN apk add --no-cache make protobuf-dev
+RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@latest && go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
+COPY ./ /apps
+RUN make
 
 FROM alpine:3.17
-COPY --from=builder /apps/collector-agent/CollectorAgent /usr/local/bin
+COPY --from=builder /apps/collector-agent /usr/local/bin/CollectorAgent
 ENTRYPOINT [ "CollectorAgent" ]

collector-agent/agent/GrpcAgent.go

@@ -321,7 +321,7 @@ func (agent *GrpcAgent) collectorActiveThreadCount(conn *grpc.ClientConn, respon
 	if activeThreadCountClient, err := client.CommandStreamActiveThreadCount(ctx); err == nil {
 		sequenceId := int32(1)
 		for {
-			agent.log.Infof("ResponseId %d", responseId)
+			agent.log.Debugf("ResponseId %d", responseId)
 			response := v1.PCmdStreamResponse{
 				ResponseId: responseId,
 				SequenceId: sequenceId,

collector-agent/makefile

@@ -5,8 +5,6 @@ TAG=${tag}
 default: server ;
 
 protoc:
-	# fixme
-	# protoc --go_out=pinpoint-grpc-idl/proto/ --go-grpc_out=pinpoint-grpc-idl/proto  --proto_path=pinpoint-grpc-idl/proto/ --go-grpc_opt=paths=source_relative pinpoint-grpc-idl/proto/v1/*.proto
 	protoc --go_out=pinpoint-grpc-idl/proto/ --go-grpc_out=pinpoint-grpc-idl/proto  --proto_path=pinpoint-grpc-idl/proto/ --go-grpc_opt=paths=source_relative v1/Annotation.proto v1/Cmd.proto v1/CustomMetric.proto v1/Service.proto v1/Span.proto v1/Stat.proto v1/ThreadDump.proto
 
 server: protoc
@@ -15,7 +13,7 @@ server: protoc
 
 clean:
 	rm -rf pinpoint-grpc-idl/proto/v1/*.go
-	rm collector-agent
+	rm -rf collector-agent
 
 docker:
 	docker build -f Dockerfile -t ${TAG}  . 

collector-agent/todo.md

@@ -1,5 +1,5 @@
 ### Todo list
-- [ ] support unique id
-- [ ] support url template
-    - [ ] collector-agent
-    - [ ] support metric
+- [-] support unique id
+- [-] support url template
+    - [-] collector-agent
+    - [-] support metric

plugins/PY/pinpointPy/Common.py

@@ -19,8 +19,8 @@
 
 # Created by eeliu at 3/5/20
 
-from . import Defines
-from . import pinpoint
+from pinpointPy import Defines
+from pinpointPy import pinpoint
 from abc import ABCMeta, abstractmethod
 
 class PinTrace(object):

plugins/PY/pinpointPy/CommonPlugin.py

@@ -15,9 +15,9 @@
 # ------------------------------------------------------------------------------
 
 
-from . import Common
-from . import Defines
-from . import pinpoint
+from pinpointPy import Common
+from pinpointPy import Defines
+from pinpointPy import pinpoint
 
 
 class PinpointCommonPlugin(Common.PinTrace):

plugins/PY/pinpointPy/Defines.py

@@ -32,7 +32,7 @@
 PP_HEADER_PINPOINT_PAPPNAME = 'Pinpoint-Pappname'
 PP_HEADER_PINPOINT_PAPPTYPE = 'Pinpoint-Papptype'
 PP_HEADER_PINPOINT_HOST = 'Pinpoint-Host'
-PP_HEADER_PINPOINT_CLIENT='Pinpoint-Client' # add for Celery Caller client-ip
+PP_HEADER_PINPOINT_CLIENT = 'Pinpoint-Client'  # add for Celery Caller client-ip
 
 PP_HEADER_NGINX_PROXY = 'Pinpoint-ProxyNginx'
 PP_HTTP_NGINX_PROXY = 'HTTP_Pinpoint-ProxyNginx'
@@ -42,61 +42,62 @@
 PP_HEADER_PINPOINT_SAMPLED = "Pinpoint-Sampled"
 PP_HTTP_PINPOINT_SAMPLED = 'HTTP_PINPOINT_SAMPLED'
 
-PP_DESTINATION='dst'
-PP_INTERCEPTOR_NAME='name'
-PP_APP_NAME='appname'
-PP_APP_ID='appid'
-PP_REQ_URI='uri'
-PP_REQ_CLIENT='client'
-PP_REQ_SERVER='server'
-PP_SERVER_TYPE='stp'
-PP_AGENT_TYPE='FT'
+PP_DESTINATION = 'dst'
+PP_INTERCEPTOR_NAME = 'name'
+PP_APP_NAME = 'appname'
+PP_APP_ID = 'appid'
+PP_REQ_URI = 'uri'
+PP_REQ_CLIENT = 'client'
+PP_REQ_SERVER = 'server'
+PP_SERVER_TYPE = 'stp'
+PP_AGENT_TYPE = 'FT'
+PP_URL_TEMPLATED = 'UT'
 
-PP_PARENT_SPAN_ID='psid'
-PP_PARENT_NAME='pname'
-PP_PARENT_TYPE='ptype'
-PP_PARENT_HOST='Ah'
+PP_PARENT_SPAN_ID = 'psid'
+PP_PARENT_NAME = 'pname'
+PP_PARENT_TYPE = 'ptype'
+PP_PARENT_HOST = 'Ah'
 
-PP_NGINX_PROXY='NP'
-PP_APACHE_PROXY='AP'
-PP_TRANSCATION_ID='tid'
-PP_SPAN_ID='sid'
-PP_NOT_SAMPLED='s0'
-PP_SAMPLED='s1'
-PP_NEXT_SPAN_ID='nsid'
-PP_ADD_EXCEPTION='EXP'
+PP_NGINX_PROXY = 'NP'
+PP_APACHE_PROXY = 'AP'
+PP_TRANSCATION_ID = 'tid'
+PP_SPAN_ID = 'sid'
+PP_NOT_SAMPLED = 's0'
+PP_SAMPLED = 's1'
+PP_NEXT_SPAN_ID = 'nsid'
+PP_ADD_EXCEPTION = 'EXP'
 
 
-PP_SQL_FORMAT='SQL'
-PP_ARGS='-1'
-PP_RETURN='14'
+PP_SQL_FORMAT = 'SQL'
+PP_ARGS = '-1'
+PP_RETURN = '14'
 PYTHON = '1700'
-PP_METHOD_CALL= '1701'
-PP_CELERY='1702'
+PP_METHOD_CALL = '1701'
+PP_CELERY = '1702'
 
 PP_REMOTE_METHOD = '9900'
 
-PP_HTTP_URL= '40'
-PP_HTTP_PARAM= '41'
-PP_HTTP_PARAM_ENTITY= '42'
-PP_HTTP_COOKIE= '45'
-PP_HTTP_STATUS_CODE= '46'
+PP_HTTP_URL = '40'
+PP_HTTP_PARAM = '41'
+PP_HTTP_PARAM_ENTITY = '42'
+PP_HTTP_COOKIE = '45'
+PP_HTTP_STATUS_CODE = '46'
 PP_HTTP_METHOD = '206'
-PP_HTTP_INTERNAL_DISPLAY=48
-PP_HTTP_IO=49
-PP_MESSAGE_QUEUE_URI=100
+PP_HTTP_INTERNAL_DISPLAY = 48
+PP_HTTP_IO = 49
+PP_MESSAGE_QUEUE_URI = 100
 
 
-PP_MYSQL= '2101'
-PP_REDIS= '8200'
-PP_REDIS_REDISSON= '8203'
-PP_REDIS_REDISSON_INTERNAL= '8204'
-PP_POSTGRESQL='2501'
-PP_MEMCACHED= '8050'
-PP_MEMCACHED_FUTURE_GET='8051'
-PP_MONGDB_EXE_QUERY= '2651'
-PP_KAFKA='8660'
-PP_KAFKA_TOPIC='140'
-PP_RABBITMQ_CLIENT='8300'
-PP_RABBITMQ_EXCHANGEKEY='130'
-PP_RABBITMQ_ROUTINGKEY='131'
+PP_MYSQL = '2101'
+PP_REDIS = '8200'
+PP_REDIS_REDISSON = '8203'
+PP_REDIS_REDISSON_INTERNAL = '8204'
+PP_POSTGRESQL = '2501'
+PP_MEMCACHED = '8050'
+PP_MEMCACHED_FUTURE_GET = '8051'
+PP_MONGDB_EXE_QUERY = '2651'
+PP_KAFKA = '8660'
+PP_KAFKA_TOPIC = '140'
+PP_RABBITMQ_CLIENT = '8300'
+PP_RABBITMQ_EXCHANGEKEY = '130'
+PP_RABBITMQ_ROUTINGKEY = '131'