diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 40ce4971..ec5fe671 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -35,7 +35,6 @@ updates: directories: - "/nodejs" - "/nodejs/packages/layer" - - "/nodejs/sample-apps/aws-sdk" registries: - codeartifact schedule: @@ -58,7 +57,6 @@ updates: # - package-ecosystem: "pip" # directories: # - "/python/src/otel" - # - "/python/sample-apps/function" # - "/python/src/otel/otel_sdk" # - "/python/src/otel/tests" # schedule: diff --git a/.github/release.yml b/.github/release.yml deleted file mode 100644 index 3fdf7612..00000000 --- a/.github/release.yml +++ /dev/null @@ -1,24 +0,0 @@ -# https://docs.github.com/en/repositories/releasing-projects-on-github/automatically-generated-release-notes - -changelog: - categories: - - - title: Collector - labels: - - go - - - title: Python - labels: - - python - - - title: Javascript - labels: - - javascript - - - title: Other Dependencies - labels: - - dependencies - - - title: Other - labels: - - '*' diff --git a/.github/workflows/ci-collector.yml b/.github/workflows/ci-collector.yml deleted file mode 100644 index 04514482..00000000 --- a/.github/workflows/ci-collector.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: "Continuous Build (Collector)" - -on: - push: - paths: - - 'collector/**' - - '.github/workflows/ci-collector.yml' - branches: - - main - pull_request: - paths: - - 'collector/**' - - '.github/workflows/ci-collector.yml' - branches: - - main - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: '~1.21.9' - - uses: actions/cache@v4 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - name: Tidy - run: make gotidy - working-directory: collector - - name: Run tests - run: make gotest - working-directory: collector - build: - runs-on: ubuntu-latest - needs: [test] - strategy: - matrix: - architecture: [ amd64, arm64 ] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: '~1.21.9' - - uses: actions/cache@v4 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - name: Build Collector Executable for ${{ matrix.architecture }} architecture - run: GOARCH=${{ matrix.architecture }} make package - working-directory: collector - - name: Get Lambda Layer amd64 architecture value - if: ${{ matrix.architecture == 'amd64' }} - run: echo LAMBDA_LAYER_ARCHITECTURE=x86 | tee --append $GITHUB_ENV - - name: Get Lambda Layer arm64 architecture value - if: ${{ matrix.architecture == 'arm64' }} - run: echo LAMBDA_LAYER_ARCHITECTURE=ARM | tee --append $GITHUB_ENV - - name: Confirm architecture of built collector - working-directory: collector/build/extensions - run: | - grep ${{ env.LAMBDA_LAYER_ARCHITECTURE }} <<< "$(file collector)" diff --git a/.github/workflows/ci-nodejs.yml b/.github/workflows/ci-nodejs.yml deleted file mode 100644 index 73707f48..00000000 --- a/.github/workflows/ci-nodejs.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: "Continuous Build (NodeJS)" - -on: - push: - paths: - - 'nodejs/**' - - '.github/workflows/ci-nodejs.yml' - branches: - - main - pull_request: - paths: - - 'nodejs/**' - - '.github/workflows/ci-nodejs.yml' - branches: - - main - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 18 - - uses: actions/cache@v4 - with: - path: ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node- - - run: npm install - working-directory: nodejs - - run: npm run lint - working-directory: nodejs - - run: npm test - working-directory: nodejs diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml deleted file mode 100644 index 6c777be1..00000000 --- a/.github/workflows/ci-python.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: "Continuous Build (Python)" - -on: - push: - paths: - - 'python/**' - - '.github/workflows/ci-python.yml' - branches: - - main - pull_request: - paths: - - 'python/**' - - '.github/workflows/ci-python.yml' - branches: - - main - -env: - AWS_REGION: us-east-1 - # Copied this CORE_REPO_SHA from - # https://github.com/open-telemetry/opentelemetry-python-contrib/blob/main/.github/workflows/test.yml#L9 - CORE_REPO_SHA: v1.19.0 - -jobs: - build: - runs-on: ubuntu-latest - - strategy: - matrix: - python: ['3.8', '3.9', '3.10', '3.11'] - - steps: - - name: Checkout this repo - uses: actions/checkout@v4 - - name: Setup Python for OTel Python SDK - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python }} - - name: Install tox testing package - working-directory: python/src - run: | - pip install tox - tox - - name: Set up Go for ADOT Collector - uses: actions/setup-go@v5 - with: - go-version: '^1.20.8' - - name: Build Python Layer which includes ADOT Collector - working-directory: python/src - run: ./run.sh -b diff --git a/.github/workflows/ci-shellcheck.yml b/.github/workflows/ci-shellcheck.yml deleted file mode 100644 index 834d93b8..00000000 --- a/.github/workflows/ci-shellcheck.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: "Continuous Build (shellcheck)" - -on: [push, pull_request] - -jobs: - shellcheck: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install shell check - run: sudo apt update && sudo apt install --assume-yes shellcheck - - - name: Run shellcheck - run: >- - find . -type f | - grep -v '.git' | - xargs -n 30 file | - grep -i shell | - awk -F':' '{print $1}' | - xargs shellcheck - diff --git a/.github/workflows/ci-terraform.yml b/.github/workflows/ci-terraform.yml deleted file mode 100644 index ae158b40..00000000 --- a/.github/workflows/ci-terraform.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: "Continuous Build (Terraform)" - -on: - push: - paths: - - '**/*.tf' - - '.github/workflows/ci-terraform.yml' - branches: - - main - pull_request: - paths: - - '**/*.tf' - - '.github/workflows/ci-terraform.yml' - branches: - - main - -jobs: - check-terraform-syntax: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: hashicorp/setup-terraform@v3 - - run: terraform fmt -check -recursive diff --git a/.github/workflows/close-stale.yaml b/.github/workflows/close-stale.yaml deleted file mode 100644 index 911e4088..00000000 --- a/.github/workflows/close-stale.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: 'Close stale issues and PRs' -on: - schedule: - - cron: "40 3 * * *" # Run daily at 3:40 AM - -permissions: - issues: write - pull-requests: write - -jobs: - stale: - runs-on: ubuntu-latest - steps: - - uses: actions/stale@v9 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-issue-message: 'This issue was marked stale. It will be closed in 30 days without additional activity.' - close-issue-message: 'Closed as inactive. Feel free to reopen if this issue is still relevant.' - - stale-pr-message: 'This PR was marked stale. It will be closed in 30 days without additional activity.' - close-pr-message: 'Closed as inactive. Feel free to reopen if this PR is still being worked on.' - - days-before-stale: 365 - days-before-close: 30 - - operations-per-run: 500 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index 8311c518..00000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,127 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: [ "main" ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ "main" ] - schedule: - - cron: '37 10 * * 2' - -jobs: - analyze: - name: Analyze - # Runner size impacts CodeQL analysis time. To learn more, please see: - # - https://gh.io/recommended-hardware-resources-for-running-codeql - # - https://gh.io/supported-runners-and-hardware-resources - # - https://gh.io/using-larger-runners - # Consider using larger runners for possible analysis time improvements. - runs-on: 'ubuntu-latest' - timeout-minutes: 360 - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - # Targets of the codeql analysis - # Each entry is composed by two elements: the language and the directory - # containing source code for that language. - target: - - language: 'go' - directory: 'go' - - language: 'javascript' - directory: 'nodejs' - - language: 'python' - directory: 'python' - - language: 'ruby' - directory: 'ruby' - - language: 'java' - directory: 'java' - - language: 'csharp' - directory: 'dotnet' - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ] - # Use only 'java' to analyze code written in Java, Kotlin or both - # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 - with: - languages: ${{ matrix.target.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - - # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - - - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v3 - with: - working-directory: ${{ matrix.target.directory }} - # There are no array literals in GHA that is why we need to use fromJson. - if: ${{ !contains(fromJson('["java", "csharp"]'), matrix.target.language) }} - - # ℹ️ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # If the Autobuild fails above, remove it and uncomment the following three lines. - # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - - # - run: | - # echo "Run, Build Application using script" - # ./location_of_script_within_repo/buildscript.sh - - - name: Set up Java - uses: actions/setup-java@v4 - with: - distribution: corretto - java-version: '11' - if: ${{ matrix.target.language == 'java' }} - - - name: build Java - uses: gradle/gradle-build-action@v3 - with: - arguments: build --no-build-cache - build-root-directory: ${{ matrix.target.directory }} - if: ${{ matrix.target.language == 'java' }} - - - name: setup dotnet - uses: actions/setup-dotnet@v4 - with: - dotnet-version: 6.x - if: ${{ matrix.target.language == 'csharp' }} - - - name: build dotnet - # Build all dotnet applications from this directory - run: find . -name '*.sln' -exec dotnet build '{}' \; - working-directory: ${{ matrix.target.directory }} - if: ${{ matrix.target.language == 'csharp' }} - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 - with: - category: "/language:${{matrix.target.language}}" diff --git a/.github/workflows/layer-publish.yml b/.github/workflows/layer-publish.yml deleted file mode 100644 index 08730240..00000000 --- a/.github/workflows/layer-publish.yml +++ /dev/null @@ -1,116 +0,0 @@ -name: Publish Lambda Layer - -on: - workflow_call: - inputs: - artifact-name: - description: 'This should correspond to a actions/upload-artifact name earlier in the build. The file name and the name of the artifact containing it must be equal.' - required: true - type: string - layer-name: - description: 'Layer name not including other parts like arch or version.' - required: true - type: string - component-version: - description: 'Version of the component included in this release. Not the same as the layer/tagged version.' - required: true - type: string - architecture: - description: '(optional) amd64 or arm64' - required: false - type: string - runtimes: - description: '(optional) a space delimited list of compatible runtimes (from https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html)' - required: false - type: string - release-group: - description: 'Release to dev or prod? "prod" yields empty value. (Default: dev)' - required: true - default: dev - type: string - aws_region: - description: 'Publish to which AWS region?' - required: true - type: string - - -permissions: - id-token: write - contents: read - -jobs: - publish_layer: - runs-on: ubuntu-latest - - steps: - - - name: Construct Layer Name - shell: bash - run: | - LAYER_NAME=${{ inputs.layer-name }} - - if [[ -n "${{ inputs.architecture }}" ]]; then - LAYER_NAME=$LAYER_NAME-${{ inputs.architecture }} - ARCH=$(echo "${{ inputs.architecture }}" | sed -r 's/amd64/x86_64/g') - else - ARCH="x86_64 arm64" - fi - echo "ARCH=$ARCH" >> $GITHUB_ENV - - if [[ -n "${{ inputs.runtimes }}" ]]; then - RUNTIMES="--compatible-runtimes ${{ inputs.runtimes }}" - fi - echo "RUNTIMES=$RUNTIMES" >> $GITHUB_ENV - - if [[ "${{ inputs.release-group }}" != "prod" ]]; then - LAYER_NAME=$LAYER_NAME-${{ inputs.release-group }} - fi - - LAYER_VERSION=$(echo "$GITHUB_REF_NAME" | sed -r 's/.*\/[^0-9\.]*//g') - LAYER_VERSION_CLEANED=$(echo "$LAYER_VERSION" | sed -r 's/\./_/g') - - LAYER_NAME=$LAYER_NAME-$LAYER_VERSION_CLEANED - echo "LAYER_NAME=$LAYER_NAME" >> $GITHUB_ENV - - echo GITHUB_ENV: - cat $GITHUB_ENV - - - name: Download built layer - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.artifact-name }} - - - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.PROD_LAMBDA_ROLE_ARN }} - role-duration-seconds: 1200 - aws-region: ${{ inputs.aws_region }} - mask-aws-account-id: false - - - name: Publish Lambda Layer - run: | - LAYER_ARN=$( - aws lambda publish-layer-version \ - --layer-name $LAYER_NAME \ - --license-info "Apache 2.0" \ - --compatible-architectures $ARCH $RUNTIMES \ - --zip-file fileb://${{ inputs.artifact-name }} \ - --query 'LayerVersionArn' \ - --output text - ) - echo "::notice ::$LAYER_ARN component-version=${{ inputs.component-version }}" - # echo "* $LAYER_ARN" >> $GITHUB_STEP_SUMMARY - - - name: Make Layer Public - run: | - LAYER_VERSION=$( - aws lambda list-layer-versions \ - --layer-name $LAYER_NAME \ - --query 'max_by(LayerVersions, &Version).Version' - ) - aws lambda add-layer-version-permission \ - --layer-name $LAYER_NAME \ - --version-number $LAYER_VERSION \ - --principal "*" \ - --statement-id publish \ - --action lambda:GetLayerVersion diff --git a/.github/workflows/publish-extend-otel-layer.yml b/.github/workflows/publish-extend-otel-layer.yml index 0c1a1839..5be423b6 100644 --- a/.github/workflows/publish-extend-otel-layer.yml +++ b/.github/workflows/publish-extend-otel-layer.yml @@ -10,7 +10,7 @@ on: - 'collector/**' - 'nodejs/**' - 'extend/**' - - 'ci-scripts/**' + - 'scripts/**' - '.github/workflows/publish-extend-otel-layer.yml' workflow_dispatch: @@ -46,7 +46,7 @@ jobs: steps: - uses: actions/checkout@v4 # Pinned SHA — bump when upstream cx-contrib lands a fix/feature we want. - # Keep in sync with CX_CONTRIB_SHA in ci-scripts/publish-sandbox.sh. + # Keep in sync with CX_CONTRIB_SHA in scripts/publish-sandbox.sh. - uses: actions/checkout@v4 with: { repository: coralogix/opentelemetry-js-contrib, ref: 3a9691a699ddd06c3644eec70bf4b50cc4217ba3, path: opentelemetry-js-contrib } - uses: actions/setup-node@v4 @@ -54,11 +54,11 @@ jobs: node-version: 22 cache: npm cache-dependency-path: nodejs/packages/layer/package-lock.json - - run: ./ci-scripts/build_nodejs_layer.sh + - run: ./scripts/build_nodejs_layer.sh - env: FILE_PATH: ./nodejs/packages/layer/build/layer.zip MAX_SIZE: 9437184 - run: ./ci-scripts/check_size.sh + run: ./scripts/check_size.sh - uses: actions/upload-artifact@v4 with: name: nodejs-layer diff --git a/.github/workflows/publish-nodejs.yml b/.github/workflows/publish-nodejs.yml deleted file mode 100644 index d8f3fdc7..00000000 --- a/.github/workflows/publish-nodejs.yml +++ /dev/null @@ -1,108 +0,0 @@ -name: Publish Node.js - -on: - workflow_dispatch: - inputs: - environment: - description: 'Actually publish or just test?' - type: environment - required: true - default: test - -jobs: - - package: - runs-on: ubuntu-24.04 - if: ${{ github.ref == 'refs/heads/coralogix-nodejs-autoinstrumentation' || inputs.environment == 'test' }} - environment: ${{ inputs.environment }} - env: - OPENTELEMETRY_JS_CONTRIB_PATH: ${{ github.workspace }}/opentelemetry-js-contrib - OPENTELEMETRY_JS_PATH: ${{ github.workspace }}/opentelemetry-js - IITM_PATH: ${{ github.workspace }}/import-in-the-middle - steps: - - uses: actions/checkout@v3 - - uses: actions/checkout@v3 - with: - path: ${{ env.OPENTELEMETRY_JS_CONTRIB_PATH }} - repository: coralogix/opentelemetry-js-contrib - ref: refs/heads/coralogix-autoinstrumentation - ssh-key: ${{ secrets.OPENTELEMETRY_CI_GITHUB_KEY }} - - uses: actions/checkout@v3 - with: - path: ${{ env.OPENTELEMETRY_JS_PATH }} - repository: coralogix/opentelemetry-js - ref: refs/heads/coralogix-autoinstrumentation - ssh-key: ${{ secrets.OPENTELEMETRY_CI_GITHUB_KEY }} - - uses: actions/checkout@v3 - with: - path: ${{ env.IITM_PATH }} - repository: coralogix/import-in-the-middle - ref: refs/heads/coralogix-autoinstrumentation - ssh-key: ${{ secrets.OPENTELEMETRY_CI_IITM_GITHUB_KEY }} - - uses: actions/setup-node@v4 - with: - node-version: 20 - - name: Build - run: ./ci-scripts/build_nodejs_layer.sh - - name: Check layer size - env: - FILE_PATH: ./nodejs/packages/layer/build/layer.zip - MAX_SIZE: 9437184 # 9MB - run: ./ci-scripts/check_size.sh - - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-access-key-id: ${{ vars.SIGNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SIGNER_AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.SIGNER_AWS_DEFAULT_REGION }} - - name: Upload package to S3 - run: aws s3 cp ./nodejs/packages/layer/build/layer.zip s3://${{ vars.SIGNER_BUCKET }}/to-be-signed/nodejs-layer.zip - - name: Get S3 version of the uploaded file - run: | - LATEST_VERSION=$(aws s3api list-object-versions --bucket ${{ vars.SIGNER_BUCKET }} --prefix to-be-signed/nodejs-layer.zip --query 'Versions[?IsLatest].[VersionId]' --output text) - echo "LATEST_VERSION=$LATEST_VERSION" >> $GITHUB_ENV - - name: Sign the package - uses: clowdhaus/aws-lambda-code-signing-action@v1.3.0 - with: - aws-region: ${{ vars.SIGNER_AWS_DEFAULT_REGION }} - source-s3-bucket: ${{ vars.SIGNER_BUCKET }} - source-s3-key: to-be-signed/nodejs-layer.zip - source-s3-version: ${{ env.LATEST_VERSION }} - destination-s3-bucket: ${{ vars.SIGNER_BUCKET }} - destination-s3-prefix: signed/ - profile-name: ${{ vars.SIGNER_PROFILE_NAME }} - max-wait-time: 60 - rename-signed-object: true - - name: Download the signed package from S3 - run: aws s3 cp s3://${{ vars.SIGNER_BUCKET }}/signed/nodejs-layer.zip ./layer.zip - - name: Upload package - uses: actions/upload-artifact@v4 - with: - name: layer.zip - path: ./layer.zip - if-no-files-found: error - retention-days: 1 - - publish-layer: - runs-on: ubuntu-22.04 - if: ${{ github.ref == 'refs/heads/coralogix-nodejs-autoinstrumentation' || inputs.environment == 'test' }} - needs: [package] - environment: ${{ inputs.environment }} - steps: - - uses: actions/checkout@v3 - - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-access-key-id: ${{ vars.LAYER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.LAYER_AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.LAYER_AWS_DEFAULT_REGION }} - - name: create target dir - run: mkdir -p target - - name: Download layer.zip - uses: actions/download-artifact@v4 - with: - name: layer.zip - path: target - - name: Publish layer - env: - LAYER_NAME: "coralogix-opentelemetry-nodejs-wrapper" - COMPATIBLE_RUNTIMES: "nodejs18.x nodejs20.x nodejs22.x nodejs24.x" - run: ./ci-scripts/publish_${{ inputs.environment }}.sh diff --git a/.github/workflows/release-layer-collector.yml b/.github/workflows/release-layer-collector.yml deleted file mode 100644 index ede4761f..00000000 --- a/.github/workflows/release-layer-collector.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: "Release Collector Lambda layer" - -on: - # (Using tag push instead of release to allow filtering by tag prefix.) - push: - tags: - - layer-collector/** - -permissions: - id-token: write - contents: read - -jobs: - build-layer: - runs-on: ubuntu-latest - strategy: - matrix: - architecture: - - amd64 - - arm64 - outputs: - COLLECTOR_VERSION: ${{ steps.save-collector-version.outputs.COLLECTOR_VERSION }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: '^1.23.1' - - name: build - run: make -C collector package GOARCH=${{ matrix.architecture }} - - uses: actions/upload-artifact@v4 - with: - name: opentelemetry-collector-layer-${{ matrix.architecture }}.zip - path: ${{ github.workspace }}/collector/build/opentelemetry-collector-layer-${{ matrix.architecture }}.zip - - name: Save Collector Version - if: ${{ matrix.architecture == 'amd64' }} - id: save-collector-version - shell: bash - # `./collector -v` output is in the form `v0.75.0` - run: | - COLLECTOR_VERSION=$( ${{ github.workspace }}/collector/build/extensions/collector -v) - echo "COLLECTOR_VERSION=$COLLECTOR_VERSION" >> $GITHUB_OUTPUT - - publish-layer: - uses: ./.github/workflows/layer-publish.yml - needs: build-layer - strategy: - matrix: - architecture: - - amd64 - - arm64 - aws_region: - - ap-northeast-1 - - ap-northeast-2 - - ap-south-1 - - ap-southeast-1 - - ap-southeast-2 - - ca-central-1 - - eu-central-1 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - sa-east-1 - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - with: - artifact-name: opentelemetry-collector-layer-${{ matrix.architecture }}.zip - layer-name: opentelemetry-collector - component-version: ${{needs.build-layer.outputs.COLLECTOR_VERSION}} - architecture: ${{ matrix.architecture }} - release-group: prod - aws_region: ${{ matrix.aws_region }} - secrets: inherit diff --git a/.github/workflows/release-layer-nodejs.yml b/.github/workflows/release-layer-nodejs.yml deleted file mode 100644 index f55e64cb..00000000 --- a/.github/workflows/release-layer-nodejs.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: "Release NodeJS Lambda Layer" - -on: - # (Using tag push instead of release to allow filtering by tag prefix.) - push: - tags: - - layer-nodejs/** - -permissions: - id-token: write - contents: read - -jobs: - build-layer: - runs-on: ubuntu-latest - outputs: - NODEJS_VERSION: ${{ steps.save-node-sdk-version.outputs.SDK_VERSION}} - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: 20 - - - name: Build - run: | - npm install - npm run build - working-directory: nodejs - - - name: Save Node SDK Version - id: save-node-sdk-version - run: | - SDK_VERSION=$(node version.js) - echo "SDK_VERSION=$SDK_VERSION" >> $GITHUB_OUTPUT - working-directory: nodejs/packages/layer/scripts - - - name: Rename zip file - run: mv layer.zip opentelemetry-nodejs-layer.zip - working-directory: nodejs/packages/layer/build - - - uses: actions/upload-artifact@v4 - name: Save assembled layer to build - with: - name: opentelemetry-nodejs-layer.zip - path: nodejs/packages/layer/build/opentelemetry-nodejs-layer.zip - - publish-layer: - uses: ./.github/workflows/layer-publish.yml - needs: build-layer - strategy: - matrix: - aws_region: - - ap-northeast-1 - - ap-northeast-2 - - ap-south-1 - - ap-southeast-1 - - ap-southeast-2 - - ca-central-1 - - eu-central-1 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - sa-east-1 - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - with: - artifact-name: opentelemetry-nodejs-layer.zip - layer-name: opentelemetry-nodejs - component-version: ${{needs.build-layer.outputs.NODEJS_VERSION}} - # architecture: - runtimes: nodejs18.x nodejs20.x - release-group: prod - aws_region: ${{ matrix.aws_region }} - secrets: inherit diff --git a/.github/workflows/release-layer-python.yml b/.github/workflows/release-layer-python.yml deleted file mode 100644 index 41f9a8e6..00000000 --- a/.github/workflows/release-layer-python.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: "Release Python Lambda Layer" - -on: - # (Using tag push instead of release to allow filtering by tag prefix.) - push: - tags: - - layer-python/** - -permissions: - id-token: write - contents: read - -jobs: - build-layer: - runs-on: ubuntu-latest - outputs: - PYTHON_OPENTELEMETRY_SDK_VERSION: ${{ steps.save-python-opentelemetry-sdk-version.outputs.PYTHON_OPENTELEMETRY_SDK_VERSION}} - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - - name: Save PYTHON_OPENTELEMETRY_SDK_VERSION - id: save-python-opentelemetry-sdk-version - run: | - cd python/src - echo "PYTHON_OPENTELEMETRY_SDK_VERSION=$(cat otel/otel_sdk/requirements.txt | grep opentelemetry-sdk | sed 's/.*==\([^ ]*\).*/\1/')" >> $GITHUB_ENV - echo "PYTHON_OPENTELEMETRY_SDK_VERSION=$PYTHON_OPENTELEMETRY_SDK_VERSION" >> $GITHUB_OUTPUT - shell: bash - - - name: Display PYTHON_OPENTELEMETRY_SDK_VERSION - run: | - echo "PYTHON_OPENTELEMETRY_SDK_VERSION=${{ env.PYTHON_OPENTELEMETRY_SDK_VERSION }}" - shell: bash - - - name: Build - run: | - cd python/src - ./build.sh - shell: bash - - - name: Show directory contents - run: | - ls -al - working-directory: python/src/build - - - uses: actions/upload-artifact@v4 - name: Save assembled layer to build - with: - name: opentelemetry-python-layer.zip - path: python/src/build/opentelemetry-python-layer.zip - - publish-layer: - uses: ./.github/workflows/layer-publish.yml - needs: build-layer - strategy: - matrix: - aws_region: - - ap-northeast-1 - - ap-northeast-2 - - ap-south-1 - - ap-southeast-1 - - ap-southeast-2 - - ca-central-1 - - eu-central-1 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - sa-east-1 - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - with: - artifact-name: opentelemetry-python-layer.zip - layer-name: opentelemetry-python - component-version: ${{needs.build-layer.outputs.PYTHON_OPENTELEMETRY_SDK_VERSION}} - release-group: prod - aws_region: ${{ matrix.aws_region }} - secrets: inherit diff --git a/README.md b/README.md index c0376df5..d1961d37 100644 --- a/README.md +++ b/README.md @@ -20,15 +20,15 @@ The collector extension (in `collector/`) is language-agnostic and powers both N | `extend/` | Extend-specific collector configs + README for consumers | | `nodejs/` | Node.js wrapper + cx-wrapper package | | `python/` | Python wrapper | -| `ci-scripts/` | Build scripts (`build_nodejs_layer.sh`, `publish-sandbox.sh`, `check_size.sh`) | -| `.github/workflows/publish-extend-otel-layer.yml` | Publish pipeline (Node.js layer today) | +| `scripts/` | Build & dev scripts (`build_nodejs_layer.sh`, `publish-sandbox.sh`, `check_size.sh`, `build-nodejs.sh`, `deploy-nodejs.sh`) | +| `.github/workflows/publish-extend-otel-layer.yml` | Extend publish pipeline (Node.js + collector layer) | | `UPSTREAM.md` | Fork point + upstream sync status | ## Publishing **Production** — push to `main` triggers `publish-extend-otel-layer.yml`, which publishes `extend-nodejs-wrapper-and-exporter-{amd64,arm64}` to account 159581800400 in `us-east-1` and `us-west-2`, org-visible to all Extend AWS accounts. -**Sandbox** — `./ci-scripts/publish-sandbox.sh {amd64|arm64}` publishes a private layer (`extend-nodejs-wrapper-and-exporter-sandbox-{arch}`) to the currently-authenticated account in `us-east-1`. Auto-clones the pinned `coralogix/opentelemetry-js-contrib` fork to `.build-cache/` on first run. +**Sandbox** — `./scripts/publish-sandbox.sh {amd64|arm64}` publishes a private layer (`extend-nodejs-wrapper-and-exporter-sandbox-{arch}`) to the currently-authenticated account in `us-east-1`. Auto-clones the pinned `coralogix/opentelemetry-js-contrib` fork to `.build-cache/` on first run. ## Consumer wiring diff --git a/RELEASE.md b/RELEASE.md deleted file mode 100644 index a39a5c52..00000000 --- a/RELEASE.md +++ /dev/null @@ -1,14 +0,0 @@ -# OpenTelemetry Lambda Layer Release Procedure - -The release process is almost entirely managed by [GitHub actions](https://github.com/open-telemetry/opentelemetry-lambda/tree/main/.github/workflows). To publish a new layer: - -1. Create a new tag for the layer to publish. For example, to create a new collector layer, the following command is used: - `git tag layer-collector/0.0.8` -2. Push the tag to [opentelemetry-lambda](https://github.com/open-telemetry/opentelemetry-lambda) repository to trigger the publish action: - `git push origin tag layer-collector/0.0.8` -3. Wait for the [release workflow](https://github.com/open-telemetry/opentelemetry-lambda/actions/workflows/release-layer-collector.yml) to finish. -4. Create a release in https://github.com/open-telemetry/opentelemetry-lambda/releases/new - * Select a the newly pushed tag - * Select the corresponding previous release - * Click "Generate Release Notes" - * Adjust the release notes. Include the ARN, list of changes and diff with previous label. diff --git a/UPSTREAM.md b/UPSTREAM.md index aed039fa..535006f7 100644 --- a/UPSTREAM.md +++ b/UPSTREAM.md @@ -2,7 +2,7 @@ This repo is a fork of [`coralogix/opentelemetry-lambda`](https://github.com/coralogix/opentelemetry-lambda) (branch `coralogix-nodejs-autoinstrumentation`), which is itself a fork of [`open-telemetry/opentelemetry-lambda`](https://github.com/open-telemetry/opentelemetry-lambda). -We also consume [`coralogix/opentelemetry-js-contrib`](https://github.com/coralogix/opentelemetry-js-contrib) (branch `coralogix-autoinstrumentation`) at build time — pinned separately in `ci-scripts/publish-sandbox.sh` and `.github/workflows/publish-extend-otel-layer.yml`. +We also consume [`coralogix/opentelemetry-js-contrib`](https://github.com/coralogix/opentelemetry-js-contrib) (branch `coralogix-autoinstrumentation`) at build time — pinned separately in `scripts/publish-sandbox.sh` and `.github/workflows/publish-extend-otel-layer.yml`. Why we fork cx-contrib: upstream OpenTelemetry has declined the Lambda-specific PRs (trigger subsystem, early-spans-on-timeout, `cx.internal.*` reconciliation attrs — see contrib#1349, contrib#1295, contrib#1309). See `~/workspace/scratch/otel-fork-research/summary.md` for the full rationale. @@ -43,7 +43,7 @@ git log 3a9691a699ddd06c3644eec70bf4b50cc4217ba3..origin/coralogix-autoinstrumen ``` When bumping, update **both** places in sync: -- `ci-scripts/publish-sandbox.sh` → `CX_CONTRIB_SHA` +- `scripts/publish-sandbox.sh` → `CX_CONTRIB_SHA` - `.github/workflows/publish-extend-otel-layer.yml` → `ref:` on the cx-contrib checkout step - This file's fork-points table diff --git a/ci-scripts/publish_layer.sh b/ci-scripts/publish_layer.sh deleted file mode 100755 index 5a7bbbdf..00000000 --- a/ci-scripts/publish_layer.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -for region in "${REGIONS[@]}"; do - if [ "$LIMITED_REGION" = true ] ; then - # shellcheck disable=SC2086 - output=$(aws lambda publish-layer-version --layer-name "${LAYER_NAME}${NAME_SUFFIX}" --compatible-runtimes $COMPATIBLE_RUNTIMES --zip-file fileb://target/layer.zip --region "$region") - else - # shellcheck disable=SC2086 - output=$(aws lambda publish-layer-version --layer-name "${LAYER_NAME}${NAME_SUFFIX}" --compatible-architectures x86_64 arm64 --compatible-runtimes $COMPATIBLE_RUNTIMES --zip-file fileb://target/layer.zip --region "$region") - fi - version=$(echo "$output" | jq -r .Version) - versionArn=$(echo "$output" | jq -r .LayerVersionArn) - if [ "$PUBLIC" = true ] ; then - aws lambda add-layer-version-permission --layer-name "${LAYER_NAME}${NAME_SUFFIX}" --principal '*' --action lambda:GetLayerVersion --version-number "$version" --statement-id public --region "$region" > /dev/null - fi - echo "$versionArn" -done \ No newline at end of file diff --git a/ci-scripts/publish_production.sh b/ci-scripts/publish_production.sh deleted file mode 100755 index 8d77771c..00000000 --- a/ci-scripts/publish_production.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -#export AWS_PROFILE="prod" # not setting profile, expecting credentials to be set in env vars during github action execution -export NAME_SUFFIX="" -export PUBLIC=true - -export REGIONS=("ap-south-1" "eu-north-1" "eu-west-3" "eu-west-2" "eu-west-1" "ap-northeast-3" "ap-northeast-2" "ap-northeast-1" "ca-central-1" "sa-east-1" "ap-southeast-1" "ap-southeast-2" "eu-central-1" "us-east-1" "us-east-2" "us-west-1" "us-west-2" "af-south-1" "ap-east-1" "ap-southeast-3" "eu-south-1" "me-south-1" "ap-south-2" "ap-southeast-4" "eu-central-2" "eu-south-2" "me-central-1" "il-central-1" "ca-west-1" "ap-southeast-5") -export LIMITED_REGION=false -# shellcheck source=/dev/null -. "${BASH_SOURCE%/*}/publish_layer.sh" - -# export REGIONS=("ca-west-1") -# export LIMITED_REGION=true -# # shellcheck source=/dev/null -# . "${BASH_SOURCE%/*}/publish_layer.sh" diff --git a/ci-scripts/publish_test.sh b/ci-scripts/publish_test.sh deleted file mode 100755 index 44a48a7d..00000000 --- a/ci-scripts/publish_test.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -#export AWS_PROFILE="research" # not setting profile, expecting credentials to be set in env vars during github action execution -export NAME_SUFFIX="-test" -export PUBLIC=false - -export REGIONS=("ap-south-1" "eu-north-1" "eu-west-3" "eu-west-2" "eu-west-1" "ap-northeast-3" "ap-northeast-2" "ap-northeast-1" "ca-central-1" "sa-east-1" "ap-southeast-1" "ap-southeast-2" "eu-central-1" "us-east-1" "us-east-2" "us-west-1" "us-west-2" "af-south-1" "ap-east-1" "ap-southeast-3" "eu-south-1" "me-south-1" "ap-south-2" "ap-southeast-4" "eu-central-2" "eu-south-2" "me-central-1" "il-central-1" "ca-west-1" "ap-southeast-5") -export LIMITED_REGION=false -# shellcheck source=/dev/null -. "${BASH_SOURCE%/*}/publish_layer.sh" - -# export REGIONS=() -# export LIMITED_REGION=true -# # shellcheck source=/dev/null -# . "${BASH_SOURCE%/*}/publish_layer.sh" diff --git a/collector/Makefile b/collector/Makefile index 3e70ca82..bf68bad0 100644 --- a/collector/Makefile +++ b/collector/Makefile @@ -2,8 +2,6 @@ include ./Makefile.Common BASE_SPACE:=$(shell pwd) BUILD_SPACE:=$(BASE_SPACE)/build -BUCKET_NAME:=lambda-artifacts-$(shell dd if=/dev/random bs=8 count=1 2>/dev/null | od -An -tx1 | tr -d ' \t\n') -LAYER_NAME:=otel-collector OTELCOL_VERSION="UNSET" # run recipe set-otelcol-version to get this GIT_SHA=$(shell git rev-parse HEAD) GOARCH ?= amd64 @@ -38,13 +36,6 @@ build: clean set-otelcol-version mkdir -p $(BUILD_SPACE)/extensions GOOS=linux GOARCH=$(GOARCH) $(GOBUILD) $(LDFLAGS) -o $(BUILD_SPACE)/extensions . -.PHONY: package -package: build - @echo Package zip file for collector extension layer - mkdir -p $(BUILD_SPACE)/collector-config - cp config* $(BUILD_SPACE)/collector-config - cd $(BUILD_SPACE) && zip -r opentelemetry-collector-layer-$(GOARCH).zip collector-config extensions - .PHONY: package-extend package-extend: build @echo Packaging Extend collector layer @@ -54,16 +45,6 @@ package-extend: build cp ../extend/collector-config-cx-arize-s3.yaml $(BUILD_SPACE)/collector-config/ cd $(BUILD_SPACE) && zip -r opentelemetry-collector-layer-$(GOARCH).zip collector-config extensions -.PHONY: publish -publish: - aws lambda publish-layer-version --layer-name $(LAYER_NAME) --zip-file fileb://$(BUILD_SPACE)/opentelemetry-collector-layer-$(GOARCH).zip --compatible-runtimes nodejs14.x nodejs16.x nodejs18.x java11 python3.8 python3.9 python3.10 python3.11 --query 'LayerVersionArn' --output text - -.PHONY: publish-layer -publish-layer: package - @echo Publishing collector extension layer... - aws lambda publish-layer-version --layer-name $(LAYER_NAME) --zip-file fileb://$(BUILD_SPACE)/opentelemetry-collector-layer-$(GOARCH).zip --compatible-runtimes nodejs14.x nodejs16.x nodejs18.x java11 python3.8 python3.9 python3.10 python3.11 --query 'LayerVersionArn' --output text - @echo OpenTelemetry Collector layer published. - .PHONY: set-otelcol-version set-otelcol-version: @OTELCOL_VERSION=$$(grep "go.opentelemetry.io/collector/otelcol v" go.mod | awk '{print $$2; exit}'); \ @@ -95,17 +76,7 @@ gofmt: .PHONY: install-tools install-tools: - cd $(TOOLS_MOD_DIR) && $(GOCMD) install github.com/client9/misspell/cmd/misspell cd $(TOOLS_MOD_DIR) && $(GOCMD) install github.com/golangci/golangci-lint/cmd/golangci-lint - cd $(TOOLS_MOD_DIR) && $(GOCMD) install github.com/google/addlicense - cd $(TOOLS_MOD_DIR) && $(GOCMD) install github.com/ory/go-acc cd $(TOOLS_MOD_DIR) && $(GOCMD) install github.com/pavius/impi/cmd/impi - cd $(TOOLS_MOD_DIR) && $(GOCMD) install github.com/tcnksm/ghr - cd $(TOOLS_MOD_DIR) && $(GOCMD) install github.com/wadey/gocovmerge - cd $(TOOLS_MOD_DIR) && $(GOCMD) install go.opentelemetry.io/build-tools/chloggen - cd $(TOOLS_MOD_DIR) && $(GOCMD) install go.opentelemetry.io/build-tools/semconvgen - cd $(TOOLS_MOD_DIR) && $(GOCMD) install golang.org/x/exp/cmd/apidiff cd $(TOOLS_MOD_DIR) && $(GOCMD) install golang.org/x/tools/cmd/goimports cd $(TOOLS_MOD_DIR) && $(GOCMD) install github.com/jcchavezs/porto/cmd/porto - cd $(TOOLS_MOD_DIR) && $(GOCMD) install go.opentelemetry.io/build-tools/multimod - cd $(TOOLS_MOD_DIR) && $(GOCMD) install go.opentelemetry.io/build-tools/crosslink diff --git a/collector/Makefile.Common b/collector/Makefile.Common index be1b8dc9..1272e4ff 100644 --- a/collector/Makefile.Common +++ b/collector/Makefile.Common @@ -29,7 +29,7 @@ fmt: .PHONY: tidy tidy: rm -fr go.sum - $(GOCMD) mod tidy -compat=1.21 + $(GOCMD) mod tidy -compat=1.25 .PHONY: lint lint: diff --git a/collector/config.yaml b/collector/config.yaml deleted file mode 100644 index de306165..00000000 --- a/collector/config.yaml +++ /dev/null @@ -1,23 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - endpoint: "localhost:4317" - http: - endpoint: "localhost:4318" - -exporters: - debug: - verbosity: detailed - -service: - pipelines: - traces: - receivers: [otlp] - exporters: [debug] - metrics: - receivers: [otlp] - exporters: [debug] - telemetry: - metrics: - address: localhost:8888 diff --git a/docs/design_proposal.md b/docs/design_proposal.md deleted file mode 100644 index dbe67243..00000000 --- a/docs/design_proposal.md +++ /dev/null @@ -1,79 +0,0 @@ -# Design: OpenTelemetry support for Lambda - -## 1. Introduction -AWS Lambda (referred to as Lambda herein) is a compute service that lets the user build serverless applications, where the user’s Lambda code runs in a sandbox environment provisioned and managed by Lambda. Since users do not have full control of the environment, running OpenTelemetry SDKs and Collector in Lambda requires additional setup. For example, launching OpenTelemetry Collector and enabling auto-instrumentation in Lambda are not as straightforward as compared to Linux. That is the motivation for why we created this proposal which aims to provide a seamless OpenTelemetry UX for Lambda users. - -### Some requirements for this proposal: -* ___[Lambda container](https://aws.amazon.com/blogs/compute/container-reuse-in-lambda/):___ -A user’s Lambda function executes in a Linux container (aka sandbox). Lambda provides the user serverless infrastructure by reusing the existing container when there are requests in queue, freezing it when no new requests, thawing the frozen container if new requests come in, and scaling up capacity by expanding the number of containers. - -* ___[Lambda Extension](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-extensions-api.html):___ -The Lambda Extensions API provides a method for launching a standalone program in a Lambda container. Using the Lambda Extensions API, the OpenTelemetry Collector can run in Lambda. - -* ___[Public Lambda layer](https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html):___ -A Lambda layer is a .zip file archive that contains libraries, a custom runtime, or other dependencies. It is natural to package the Collector in a layer and share with users. For some languages like Python, the UX can be improved by the public SDK layer but not for all languages. - -## 2. Technical Challenges -There are additional limitations when running OpenTelemetry in Lambda. Here is a list of some technical challenges we need to address in development. -### 2.1 Lambda freeze -Lambda freezes the container if there are no new requests coming. When it occurs all processes in the container will be frozen. See details in [cgroup freezer](https://www.kernel.org/doc/Documentation/cgroup-v1/freezer-subsystem.txt). If the process uses a timer to flush the buffered data, the timer won’t fire until the container thaws. The interval between the frozen and the thaw state is unpredictable, ranging from seconds to hours. Therefore, the data buffered in the OpenTelemetry SDK and Collector will have unpredictable latency. - -In the long run, we hope Lambda provides an IDLE event callback, which should be around 60 seconds. After Lambda realizes it has frozen for a while, it triggers this callback so the user has a chance to flush the data in the buffer. Lambda does not know if there is a new request coming or that it will freeze or thaw. So, an IDLE event is like a wake-up event once the container has been frozen more than 60 seconds. - -The short term solution is to flush data out before the Lambda function returns, because it ensures that the Lambda container will not be frozen during an invocation. To achieve this solution: -1. The SDK side needs to call [force_flush()](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#forceflush) at the end of Lambda instrumentation. -2. The Collector needs to remove all processors from config to keep one thread only from the receiver to the exporter. In this way, telemetry data is flushed to the backend synchronously at the end of every invocation. The side effect is slightly increased invocation time. - -![Architect](https://user-images.githubusercontent.com/66336933/113330107-3e8b3f80-92d3-11eb-826c-6110773096b5.png) -### 2.2 Memory consumption -For the best performance experience, Lambda provides the memory used metric, which is used as a high-water mark. This metric is equal to cgroup memory.max_usage_in_bytes, including RSS + CACHE. The page CACHE size is approximately equal to the size of dependencies. This means the more dependencies added in Lambda, the higher the memory consumption is in Lambda metrics. Right now the size of OpenTelemetry Collector-contrib is over 100MB and this size can increase as the RSS size (currently at 10MB) grows. - -### 2.3 Layer size limit -The layer size limit is a hard limit specifying that the total unzipped size of the function and all layers can't exceed the unzipped deployment package size limit of 250 MB. Due to this constraint, we have to strip down the size of the Collector layer by removing unnecessary components. - -With the exception of the Lambda layer, we can keep an eye on [Lambda Container Image](https://docs.aws.amazon.com/lambda/latest/dg/images-create.html), a new feature that can package Lambda function code and dependencies as a container image by Docker CLI. It brings benefits such as higher volume of dependencies, but now it still has limits that cannot add/update Lambda functions and layers once the image is built. It is not possible to provide the finished image to the user directly. Lambda Container Image is still not a replacement of Lambda layer. Furthermore, it does not change the way of memory consumption in Lambda, the larger dependencies need users to apply for higher performance Lambda containers. - -### 2.4 Container platform -For programming languages such as Python, building deployment packages in a local development environment is risky. The local environment is often different from the Lambda container environment in meaningful ways. Python wheels with compiled binaries, for example, may work on Mac but fail on Linux (typically with Python gRPC). The solution is building the SDK layer in a Docker Lambda image. Languages which support cross-compiling, such as Java or Go, don't have to be built in docker. -### 2.5 Auto instrumentation -It's important to clarify that auto-instrumentation is not library instrumentation; auto-instrumentation does not require any code changes within the user's application. Refer to the OpenTelemetry doc for [Python](https://opentelemetry-python.readthedocs.io/en/stable/examples/auto-instrumentation/README.html) and [Java](https://opentelemetry.io/docs/java/getting_started/#automatic-instrumentation). Enabling OpenTelemetry auto-instrumentation in Lambda is non-trivial and is language-specific. For more details, refer to [Lambda runtime modification](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-modify.html). Need to mention is right now not all languages support auto-instrumentation in OpenTelemetry. - -## 3. Implementation -### 3.1 Three wrappers -We can have three layers of wrapper on a user's Lambda function: ___Lambda function instrumentation___, ___execution wrapper___ and ___Lambda EXEC Wrapper___. Each one can add new functionality by wrapping the previous one, to further improve user experience. -![wrappers](https://user-images.githubusercontent.com/66336933/113330096-3af7b880-92d3-11eb-89bf-580ed8614807.png) -1. ___Lambda function instrumentation___: -Lambda function instrumentation is the first step of OpenTelemetry Lambda support, which is wrapping the Lambda handler function with beginSpan() and endSpan(), adding OpenTelemetry attributes by following [trace](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/faas.md) and [resource](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/semantic_conventions/faas.md) FaaS spec. Java and Python implementations are in place, other languages can refer to the code for them. -2. ___Lambda execution wrapper___: -Lambda execution wrapper uses reflection to dynamically decorate the user’s original Lambda function. It can add functionality to a user's existing Lambda function without changing source code, but the user must change the Lambda handler name and add the original handler path as an environment variable. With OpenTelemetry, the decoration can be anything, such as initializing OpenTelemetry components, calling instrumentation, or even implementing instrumentation directly. If desired, a developer can also merge Lambda function instrumentation and Lambda execution wrapper into one wrapper. -See [Java example](https://github.com/open-telemetry/opentelemetry-java-instrumentation/pull/1471/files) (decorate Lambda instrumentation) and [Python example](https://github.com/open-telemetry/opentelemetry-lambda/blob/main/python/src/otel/otel_sdk/otel_wrapper.py) (initialize OpenTelemetry components for auto-instrumentation). -3. ___Lambda EXEC Wrapper(Lambda native wrapper)___: -Lambda has a native [Wrapper scripts](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-modify.html#runtime-wrapper) feature which is triggered by environment variable AWS_LAMBDA_EXEC_WRAPPER. The Lambda EXEC Wrapper can couple with an execution wrapper to improve the UX, see [an example in Python](https://github.com/open-telemetry/opentelemetry-lambda/blob/main/python/src/otel/otel_sdk/otel-instrument). By it, users no need to change Lambda handler name and add the original handler path as an environment variable - -### 3.2 OpenTelemetry Collector extension Lambda layer -Unless using the SDK in-process exporter, the user must run OpenTelemetry Collector in Lambda sandbox to forward telemetry data to the backend service. Wrapped by Lambda Extensions API, Collector can run in Lambda sandbox independently. We should support that in the OpenTelemetry Lambda Repo. - -In previous technical challenges we mentioned two things: Memory consumption and size limit. Both can be solved by a smaller Collector size. At the moment OpenTelemetry Collector contrib binary is over 100MB, all of OpenTelemetry Collector components are built into it. But in a real case, an user may use only a few components. So, when we develop the OpenTelemetry collector extension layer, an ideal solution is to provide a build the Collector layer on demand functionality, by picking up the components used in Collector config. Or, at least provide an interface for dynamically mounting a cherry-pick of Collector components in Lambda extension layer. For the prototype please refer to [aws-observability/aws-otel-lambda](https://github.com/aws-observability/aws-otel-lambda/) - -### 3.3 OpenTelemetry SDK layer -As a prototype, Python is implemented in [current Repo](https://github.com/open-telemetry/opentelemetry-lambda/tree/main/python). - -We don’t provide SDK public layers for every language. This is required in the following two cases: -Languages with auto-instrumentation support, such as: -* Python -* Java - -Scripting languages such as: -* Python -* Node.js -* Ruby - -At the moment only Python and Java support auto-instrumentation. Other languages such as Go, .Net support library instrumentation only. For these languages, users must explicitly add OpenTelemetry SDK dependencies and instrumentation code, then re-compile the Lambda application to enable library instrumentation. In such cases we would not provide OpenTelemetry SDK layer because it does not help in compilation and deployment. - -Java SDK supports auto-instrumentation through javaagent, which will noticeably increase the program’s start time. In performance testing we see around 40 seconds for Lambda cold start if using OpenTelemetry javaagent. This is a concern in terms of providing a public Lambda layer for Java. We do see some users asking for OpenTelemetry Java auto-instrumentation in Lambda and aren’t concerned about long cold start times. For this reason, we still provide a public Lambda layer for Java and remind users of the impact in documentation. - -Even if we don't provide an SDK layer for a language, we still need to provide a sample demonstrating how to use it. The sample would refer to the OpenTelemetry SDK dependencies directly and consume the public Collector Lambda layer. For example, for time-sensitive users we will provide a sample of Java library instrumentation, as a complement of Java(agent based) auto-instrumentation layer. - -## 4. CI/CD convention -Each sample application and layer source folder must have a one-click script `run.sh`, to build and deploy OpenTelemetry Lambda application in a uniform manner. It is not only providing a good user experience but also for building a generic CI/CD workflow, github action can simply run tests and publish Lambda layer no matter what the stack tool(Terraform, CloudFormation), or language(Java, Python) the sample application is. -See an example for publishing [Python layer](../python/src/). \ No newline at end of file diff --git a/extend/README.md b/extend/README.md index 3299f073..79917205 100644 --- a/extend/README.md +++ b/extend/README.md @@ -43,7 +43,7 @@ Required env vars: ## Build (pending workflow setup) -Follows upstream: `./ci-scripts/build_nodejs_layer.sh` — requires a sibling checkout of `coralogix/opentelemetry-js-contrib` (branch `coralogix-autoinstrumentation`) set via `OPENTELEMETRY_JS_CONTRIB_PATH`. See `.github/workflows/publish-nodejs.yml` for the published flow. +Follows upstream: `./scripts/build_nodejs_layer.sh` — requires a sibling checkout of `coralogix/opentelemetry-js-contrib` (branch `coralogix-autoinstrumentation`) set via `OPENTELEMETRY_JS_CONTRIB_PATH`. See `.github/workflows/publish-extend-otel-layer.yml` for the published flow. **Extend-specific follow-ups** (tracked in DEVOPS-2394): - [ ] Add GitHub Actions workflow to publish to Extend AWS accounts diff --git a/extend/plan-publish.md b/extend/plan-publish.md index 83786f7d..cf54d238 100644 --- a/extend/plan-publish.md +++ b/extend/plan-publish.md @@ -21,7 +21,7 @@ Plan scope: build + publish pipeline only. Consumer wiring (`extend-cdk-lib` `No | Architectures | amd64 + arm64 | Both required by consumers | | Regions | us-east-1, us-west-2 | Matches Extend's current Lambda footprint | | AWS account | 159581800400 (shared root) | Same as helloextend/layers | -| Auth | GitHub OIDC → `PROD_LAMBDA_ROLE_ARN` | Reuses existing `.github/workflows/layer-publish.yml` | +| Auth | GitHub OIDC → `PROD_LAMBDA_ROLE_ARN` | Reuses existing publish workflow pattern | | Trigger | Push to `main` | Simplest versioning; no tags needed | | Runners | Blacksmith (`blacksmith-2vcpu-ubuntu-2404`) | Extend standard | | Node.js wrapper source | Coralogix forks (public; no PAT) | `opentelemetry-js-contrib` + `opentelemetry-js` @ `coralogix-autoinstrumentation`; + `coralogix/import-in-the-middle` | @@ -38,7 +38,7 @@ Plan scope: build + publish pipeline only. Consumer wiring (`extend-cdk-lib` `No Already landed on branch: - `extend/collector-config-cx-only.yaml`, `collector-config-cx-arize.yaml`, `collector-config-cx-arize-s3.yaml` - `extend/README.md` with consumer contract -- Upstream's reusable `layer-publish.yml` (OIDC, `PROD_LAMBDA_ROLE_ARN`, `configure-aws-credentials@v4`) — **reuse, don't duplicate** +- Upstream's reusable publish workflow pattern (OIDC, `PROD_LAMBDA_ROLE_ARN`, `configure-aws-credentials@v4`) — **reuse, don't duplicate** - `collector/Makefile` `package` target already globs `config*` into `build/collector-config/` and zips with binary Outstanding (blockers): @@ -86,7 +86,7 @@ package-extend: build ### 4. Node.js wrapper build — reuse upstream script -`ci-scripts/build_nodejs_layer.sh` already builds `nodejs/packages/layer/build/layer.zip`. Needs sibling checkouts + env vars (see `publish-nodejs.yml:17-43`): +`scripts/build_nodejs_layer.sh` already builds `nodejs/packages/layer/build/layer.zip`. Needs sibling checkouts + env vars (see `publish-extend-otel-layer.yml` `build-nodejs` job): - `OPENTELEMETRY_JS_CONTRIB_PATH` → `coralogix/opentelemetry-js-contrib@coralogix-autoinstrumentation` - `OPENTELEMETRY_JS_PATH` → `coralogix/opentelemetry-js@coralogix-autoinstrumentation` @@ -123,7 +123,7 @@ on: - 'collector/**' - 'nodejs/**' - 'extend/**' - - 'ci-scripts/**' + - 'scripts/**' - '.github/workflows/publish-extend-otel-layer.yml' workflow_dispatch: @@ -163,11 +163,11 @@ jobs: with: { repository: coralogix/import-in-the-middle, ref: coralogix-autoinstrumentation, path: import-in-the-middle } - uses: actions/setup-node@v4 with: { node-version: 20 } - - run: ./ci-scripts/build_nodejs_layer.sh + - run: ./scripts/build_nodejs_layer.sh - env: FILE_PATH: ./nodejs/packages/layer/build/layer.zip MAX_SIZE: 9437184 - run: ./ci-scripts/check_size.sh + run: ./scripts/check_size.sh - uses: actions/upload-artifact@v4 with: name: nodejs-layer @@ -212,14 +212,14 @@ No `add-layer-version-permission` step — layer stays private. ### 7. Size gate -`ci-scripts/check_size.sh` runs inside `build-nodejs` (9 MB cap on nodejs zip, upstream default). Lambda layer hard limit is 250 MB unzipped across all layers — collector binary ~30-50 MB + nodejs ~9 MB leaves ample headroom. +`scripts/check_size.sh` runs inside `build-nodejs` (9 MB cap on nodejs zip, upstream default). Lambda layer hard limit is 250 MB unzipped across all layers — collector binary ~30-50 MB + nodejs ~9 MB leaves ample headroom. ## Verification End-to-end, on a throwaway branch: 1. **Local build per arch** — `make -C collector package-extend GOARCH=amd64` and `GOARCH=arm64`; inspect `build/opentelemetry-collector-layer-*.zip` has `extensions/collector` binary + all 3 configs in `collector-config/` with `config.yaml` as cx-only contents -2. **Local nodejs build** — clone the 3 coralogix forks as siblings, run `./ci-scripts/build_nodejs_layer.sh`, confirm `nodejs/packages/layer/build/layer.zip` ≤ 9 MB and contains `nodejs/node_modules/` + `otel-handler` +2. **Local nodejs build** — clone the 3 coralogix forks as siblings, run `./scripts/build_nodejs_layer.sh`, confirm `nodejs/packages/layer/build/layer.zip` ≤ 9 MB and contains `nodejs/node_modules/` + `otel-handler` 3. **Workflow dry-run** — push to a throwaway branch, trigger `workflow_dispatch`, confirm `build-collector` + `build-nodejs` succeed and artifacts upload 4. **Publish dry-run** — temporarily scope `package-and-publish` to a single arch/region, confirm `LayerVersionArn` prints in the job log and `aws lambda list-layer-versions --layer-name extend-otel-lambda-arm64 --region us-east-1` shows the version 5. **Consumer smoke** — wire one low-traffic Lambda via `extend-cdk-lib` `NodeLambdaBuilder` to the new layer ARN, confirm traces land in CX (cx-only) and Arize + S3 (cx-arize-s3) diff --git a/nodejs/.commitlintrc.yml b/nodejs/.commitlintrc.yml deleted file mode 100644 index 1a5e5ac7..00000000 --- a/nodejs/.commitlintrc.yml +++ /dev/null @@ -1,29 +0,0 @@ -extends: - - '@commitlint/config-conventional' -rules: - header-max-length: [1, 'always', 72] - type-enum: - - 2 - - always - - - ci - - feat - - fix - - docs - - style - - refactor - - perf - - test - - revert - - chore -help: | - **Possible types**: - `ci`: Changes to our CI configuration files and scripts (example scopes: Travis, Circle CI, BrowserStack, SauceLabs) - `feat`: Adds a new feature. - `fix`: Solves a bug. - `docs`: Adds or alters documentation. (example scopes: readme, worker, code_of_conduct, contributors) - `style`: Improves formatting, white-space. - `refactor`: Rewrites code without feature, performance or bug changes. - `perf`: Improves performance. - `test`: Adds or modifies tests. (example scopes: functionals, unit-tests) - `revert`: Changes that reverting other changes - `chore`: No production code change. Updating grunt tasks etc; diff --git a/nodejs/.editorconfig b/nodejs/.editorconfig deleted file mode 100644 index f68e97b0..00000000 --- a/nodejs/.editorconfig +++ /dev/null @@ -1,11 +0,0 @@ -# http://editorconfig.org - -root = true - -[*] -indent_style = space -indent_size = 2 -end_of_line = lf -charset = utf-8 -trim_trailing_whitespace = true -insert_final_newline = true diff --git a/nodejs/.gitattributes b/nodejs/.gitattributes deleted file mode 100644 index 7d65e849..00000000 --- a/nodejs/.gitattributes +++ /dev/null @@ -1,39 +0,0 @@ -## This .gitattributes file automatically formats the EOL character in certain filetypes within the repository - -## Source code -# JavaScript, TypeScript, c, and h source files -*.js text eol=lf -*.ts text eol=lf -*.h text eol=lf diff=cpp -*.c text eol=lf diff=cpp - -# Shell scripts -*.sh text eol=lf -*.bash text eol=lf - -# Windows batch and PowerShell scripts -*.bat text eol=crlf -*.cmd text eol=crlf -*.ps1 text eol=crlf - -##### Other file types ##### - -## Text files and documentation -*.txt text -README* text -RELEASING* text -CHANGELOG* text -CONTRIBUTING* text -INSTALL* text -LICENSE* text - -## Non-text documentation -*.html text diff=html -*.pdf binary -*.json text -*.rtf binary - -## Git Properties -.gitignore text -.gitmodules text -.gitattributes text diff --git a/nodejs/.npmignore b/nodejs/.npmignore deleted file mode 100644 index 9505ba94..00000000 --- a/nodejs/.npmignore +++ /dev/null @@ -1,4 +0,0 @@ -/bin -/coverage -/doc -/test diff --git a/nodejs/README.md b/nodejs/README.md index acc07920..6d6162b4 100644 --- a/nodejs/README.md +++ b/nodejs/README.md @@ -40,19 +40,11 @@ To build the layer and sample applications run the command below from the root of the application: ```sh -./dev/build-nodejs.sh +./scripts/build-nodejs.sh ``` -This is a thin wrapper over `./ci-scripts/build_nodejs_layer.sh` that +This is a thin wrapper over `./scripts/build_nodejs_layer.sh` that will clone the forked dependencies if the paths indicated by the previously mentioned environment variables are empty, then use that script to download all dependencies and compile all code. The layer zip file will be present at `./packages/layer/build/layer.zip`. - -## Sample applications - -Sample applications are provided to show usage of the above layer. - -- Application using AWS SDK - shows using the wrapper with an application using AWS SDK without code change. - - [Using layer built from source](./integration-tests/aws-sdk) - - [WIP] [Using OTel Public Layer](./sample-apps/aws-sdk) diff --git a/nodejs/eslint.config.js b/nodejs/eslint.config.js deleted file mode 100644 index e97ae8b6..00000000 --- a/nodejs/eslint.config.js +++ /dev/null @@ -1,44 +0,0 @@ -module.exports = { - plugins: [ - "@typescript-eslint", - "header" - ], - extends: [ - "../../node_modules/gts", - ], - parser: "@typescript-eslint/parser", - parserOptions: { - "project": "./tsconfig.json" - }, - rules: { - "@typescript-eslint/no-this-alias": "off", - "eqeqeq": "off", - "prefer-rest-params": "off", - "@typescript-eslint/naming-convention": [ - "error", - { - "selector": "memberLike", - "modifiers": ["private", "protected"], - "format": ["camelCase"], - "leadingUnderscore": "require" - } - ], - "@typescript-eslint/no-inferrable-types": ["error", { ignoreProperties: true }], - "arrow-parens": ["error", "as-needed"], - "prettier/prettier": ["error", { "singleQuote": true, "arrowParens": "avoid" }], - "@typescript-eslint/no-require-imports": "off", - }, - overrides: [ - { - "files": ["test/**/*.ts"], - "rules": { - "no-empty": "off", - "@typescript-eslint/ban-ts-ignore": "off", - "@typescript-eslint/no-empty-function": "off", - "@typescript-eslint/no-explicit-any": "off", - "@typescript-eslint/no-unused-vars": "off", - "@typescript-eslint/no-var-requires": "off" - } - } - ] -}; diff --git a/nodejs/lerna.json b/nodejs/lerna.json deleted file mode 100644 index 99063669..00000000 --- a/nodejs/lerna.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "version": "0.0.0", - "changelog": { - "repo": "open-telemetry/opentelemetry-lambda", - "labels": { - "breaking": ":boom: Breaking Change", - "bug": ":bug: (Bug Fix)", - "core": ":wrench: Core", - "document": ":books: (Refine Doc)", - "enhancement": ":rocket: (Enhancement)", - "feature-request": ":sparkles: (Feature)", - "internal": ":house: Internal" - }, - "cacheDir": ".changelog", - "ignoreCommitters": ["github-actions"] - } -} diff --git a/nodejs/sample-apps/aws-sdk/.eslintignore b/nodejs/sample-apps/aws-sdk/.eslintignore deleted file mode 100644 index 378eac25..00000000 --- a/nodejs/sample-apps/aws-sdk/.eslintignore +++ /dev/null @@ -1 +0,0 @@ -build diff --git a/nodejs/sample-apps/aws-sdk/.eslintrc.js b/nodejs/sample-apps/aws-sdk/.eslintrc.js deleted file mode 100644 index f726f3be..00000000 --- a/nodejs/sample-apps/aws-sdk/.eslintrc.js +++ /dev/null @@ -1,7 +0,0 @@ -module.exports = { - "env": { - "mocha": true, - "node": true - }, - ...require('../../eslint.config.js') -} diff --git a/nodejs/sample-apps/aws-sdk/README.md b/nodejs/sample-apps/aws-sdk/README.md deleted file mode 100644 index 57ad4a51..00000000 --- a/nodejs/sample-apps/aws-sdk/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# AWS SDK Sample Application - -**This Sample App is a work-in-progress because it depends on the OpenTelemetry public layer. The public layer will not be published - -This sample application demonstrates usage of the AWS SDK. To try it out, make sure the collector and nodejs Lambda -layers are built. - -In [collector](../../../collector), run `make package`. -In [nodejs](../../), run `npm install`. - -Then, run `terraform init` and `terraform apply`. The lambda function will be initialized and the URL for an API Gateway invoking the Lambda -will be displayed at the end. Send a request to the URL in a browser or using curl to execute the function. Then, -navigate to the function's logs [here](https://console.aws.amazon.com/cloudwatch/home?region=us-east-1#logStream:group=%252Faws%252Flambda%252Fhello-nodejs). -You will see a log stream with an event time corresponding to when you issued the request - open it and you can find -information about the exported spans in the log stream. diff --git a/nodejs/sample-apps/aws-sdk/config.yaml b/nodejs/sample-apps/aws-sdk/config.yaml deleted file mode 100644 index 938fccc9..00000000 --- a/nodejs/sample-apps/aws-sdk/config.yaml +++ /dev/null @@ -1,44 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - http: - -exporters: - logging: - loglevel: debug - otlp: - endpoint: "otel-traces.coralogix.com:443" - headers: - "Authorization": "Bearer " - "CX-Application-Name": "appname" - "CX-Subsystem-Name": "subsystem" - ApplicationName: "appname" - ApiName: "subsystem" - otlp/metrics: - endpoint: "otel-metrics.coralogix.com:443" - headers: - "Authorization": "Bearer " - "CX-Application-Name": "appname" - "CX-Subsystem-Name": "subsystem" - ApplicationName: "appname" - ApiName: "subsystem" - otlp/logs: - endpoint: "otel-logs.coralogix.com:443" - headers: - "Authorization": "Bearer " - "CX-Application-Name": "appname" - "CX-Subsystem-Name": "subsystem" - ApplicationName: "appname" - ApiName: "subsystem" -service: - pipelines: - metrics: - receivers: [otlp] - exporters: [logging, otlp/metrics] - logs: - receivers: [otlp] - exporters: [logging, otlp/logs] - traces: - receivers: [otlp] - exporters: [logging, otlp] \ No newline at end of file diff --git a/nodejs/sample-apps/aws-sdk/deploy/wrapper/main.tf b/nodejs/sample-apps/aws-sdk/deploy/wrapper/main.tf deleted file mode 100644 index c4f4973b..00000000 --- a/nodejs/sample-apps/aws-sdk/deploy/wrapper/main.tf +++ /dev/null @@ -1,52 +0,0 @@ -module "hello-lambda-function" { - source = "terraform-aws-modules/lambda/aws" - version = ">= 2.24.0" - - architectures = compact([var.architecture]) - function_name = var.name - handler = "index.handler" - runtime = "nodejs16.x" - - create_package = false - local_existing_package = "${path.module}/../../build/function.zip" - - memory_size = 384 - timeout = 20 - - layers = compact([ - var.collector_layer_arn, - var.sdk_layer_arn - ]) - - environment_variables = { - AWS_LAMBDA_EXEC_WRAPPER = "/opt/otel-handler" - OTEL_LOG_LEVEL = "DEBUG" - OTEL_EXPORTER_OTLP_ENDPOINT = "http://localhost:4318/" - OTEL_TRACES_SAMPLER = "always_on" - } - - tracing_mode = var.tracing_mode - - attach_policy_statements = true - policy_statements = { - s3 = { - effect = "Allow" - actions = [ - "s3:ListAllMyBuckets" - ] - resources = [ - "*" - ] - } - } -} - -module "api-gateway" { - source = "../../../../../utils/terraform/api-gateway-proxy" - - name = var.name - function_name = module.hello-lambda-function.lambda_function_name - function_invoke_arn = module.hello-lambda-function.lambda_function_invoke_arn - enable_xray_tracing = var.tracing_mode == "Active" -} - diff --git a/nodejs/sample-apps/aws-sdk/deploy/wrapper/outputs.tf b/nodejs/sample-apps/aws-sdk/deploy/wrapper/outputs.tf deleted file mode 100644 index 5de8044c..00000000 --- a/nodejs/sample-apps/aws-sdk/deploy/wrapper/outputs.tf +++ /dev/null @@ -1,7 +0,0 @@ -output "api-gateway-url" { - value = module.api-gateway.api_gateway_url -} - -output "function_role_name" { - value = module.hello-lambda-function.lambda_role_name -} diff --git a/nodejs/sample-apps/aws-sdk/deploy/wrapper/variables.tf b/nodejs/sample-apps/aws-sdk/deploy/wrapper/variables.tf deleted file mode 100644 index 4801bf5e..00000000 --- a/nodejs/sample-apps/aws-sdk/deploy/wrapper/variables.tf +++ /dev/null @@ -1,29 +0,0 @@ -variable "name" { - type = string - description = "Name of created function and API Gateway" - default = "hello-nodejs-awssdk-us-east-2" -} - -variable "collector_layer_arn" { - type = string - description = "ARN for the Lambda layer containing the OpenTelemetry collector extension" - // TODO(anuraaga): Add default when a public layer is published. -} - -variable "sdk_layer_arn" { - type = string - description = "ARN for the Lambda layer containing the OpenTelemetry NodeJS Wrapper" - // TODO(anuraaga): Add default when a public layer is published. -} - -variable "tracing_mode" { - type = string - description = "Lambda function tracing mode" - default = "PassThrough" -} - -variable "architecture" { - type = string - description = "Lambda function architecture, valid values are arm64 or x86_64" - default = "x86_64" -} diff --git a/nodejs/sample-apps/aws-sdk/package.json b/nodejs/sample-apps/aws-sdk/package.json deleted file mode 100644 index 6346ee93..00000000 --- a/nodejs/sample-apps/aws-sdk/package.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "name": "@opentelemetry-samples/lambda-awssdk", - "version": "0.0.0", - "private": true, - "description": "Sample application for AWS Lambda using AWS SDK v2.", - "main": "build/src/index.js", - "types": "build/src/index.d.ts", - "repository": "open-telemetry/opentelemetry-lambda", - "scripts": { - "clean": "rimraf build/*", - "lint": "ESLINT_USE_FLAT_CONFIG=false eslint . --ext .ts", - "lint:fix": "ESLINT_USE_FLAT_CONFIG=false eslint . --ext .ts --fix", - "build": "npm run clean && npm run compile && npm run postcompile", - "compile": "tsc -p .", - "postcompile": "copyfiles 'package*.json' build/src/ && copyfiles 'config.yaml' build/src/ && npm install --production --ignore-scripts --prefix build/src/ && cd build/src && bestzip ../function.zip *" - }, - "keywords": [ - "opentelemetry", - "awslambda", - "nodejs", - "tracing", - "profiling", - "instrumentation" - ], - "author": "OpenTelemetry Authors", - "license": "Apache-2.0", - "engines": { - "node": ">=16.0.0" - }, - "files": [ - "build/src/**/*.js", - "build/src/**/*.d.ts", - "doc", - "LICENSE", - "README.md" - ], - "devDependencies": { - "@types/aws-lambda": "8.10.145", - "@types/node": "22.6.1", - "bestzip": "2.2.1", - "copyfiles": "2.4.1", - "rimraf": "6.0.1", - "ts-node": "10.9.2", - "typescript": "5.6.2" - }, - "dependencies": { - "aws-sdk": "2.1691.0" - } -} diff --git a/nodejs/sample-apps/aws-sdk/src/index.ts b/nodejs/sample-apps/aws-sdk/src/index.ts deleted file mode 100644 index c1f82b4c..00000000 --- a/nodejs/sample-apps/aws-sdk/src/index.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { - APIGatewayProxyEvent, - APIGatewayProxyResult, - Context, -} from 'aws-lambda'; - -import AWS from 'aws-sdk'; -//import fetch from 'node-fetch'; - -const s3 = new AWS.S3(); -var docClient = new AWS.DynamoDB.DocumentClient(); -var params = { - TableName: "test", - Item: { - "id": "1", - "hello": "hey" - } -}; - -// eslint-disable-next-line @typescript-eslint/no-unused-vars -exports.handler = async (_event: APIGatewayProxyEvent, _context: Context) => { - console.info('Serving lambda request.'); - - docClient.put(params, function (err, data) { - if (err) { - console.error("Unable to put", JSON.stringify(err, null, 2)); - } else { - console.log("put succeeded:"); - } - }); - - //const responseOne = await fetch('https://google.com/123'); - //console.log(await responseOne.json()) - const result = await s3.listBuckets().promise(); - - const response: APIGatewayProxyResult = { - statusCode: 200, - body: `Hello lambda - found ${result.Buckets?.length || 0} buckets`, - }; - return response; -}; diff --git a/nodejs/sample-apps/aws-sdk/tsconfig.json b/nodejs/sample-apps/aws-sdk/tsconfig.json deleted file mode 100644 index 4078877c..00000000 --- a/nodejs/sample-apps/aws-sdk/tsconfig.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "../../tsconfig.base", - "compilerOptions": { - "rootDir": ".", - "outDir": "build" - }, - "include": [ - "src/**/*.ts", - "test/**/*.ts" - ] -} diff --git a/python/sample-apps/aws-sdk/deploy/wrapper/main.tf b/python/sample-apps/aws-sdk/deploy/wrapper/main.tf deleted file mode 100644 index 0470841f..00000000 --- a/python/sample-apps/aws-sdk/deploy/wrapper/main.tf +++ /dev/null @@ -1,48 +0,0 @@ -module "hello-lambda-function" { - source = "terraform-aws-modules/lambda/aws" - version = ">= 2.24.0" - - architectures = compact([var.architecture]) - function_name = var.name - handler = "lambda_function.lambda_handler" - runtime = var.runtime - - create_package = false - local_existing_package = "${path.module}/../../../build/function.zip" - - memory_size = 384 - timeout = 20 - - layers = compact([ - var.collector_layer_arn, - var.sdk_layer_arn - ]) - - environment_variables = { - AWS_LAMBDA_EXEC_WRAPPER = "/opt/otel-instrument" - } - - tracing_mode = var.tracing_mode - - attach_policy_statements = true - policy_statements = { - s3 = { - effect = "Allow" - actions = [ - "s3:ListAllMyBuckets" - ] - resources = [ - "*" - ] - } - } -} - -module "api-gateway" { - source = "../../../../../utils/terraform/api-gateway-proxy" - - name = var.name - function_name = module.hello-lambda-function.lambda_function_name - function_invoke_arn = module.hello-lambda-function.lambda_function_invoke_arn - enable_xray_tracing = var.tracing_mode == "Active" -} diff --git a/python/sample-apps/aws-sdk/deploy/wrapper/outputs.tf b/python/sample-apps/aws-sdk/deploy/wrapper/outputs.tf deleted file mode 100644 index 5de8044c..00000000 --- a/python/sample-apps/aws-sdk/deploy/wrapper/outputs.tf +++ /dev/null @@ -1,7 +0,0 @@ -output "api-gateway-url" { - value = module.api-gateway.api_gateway_url -} - -output "function_role_name" { - value = module.hello-lambda-function.lambda_role_name -} diff --git a/python/sample-apps/aws-sdk/deploy/wrapper/variables.tf b/python/sample-apps/aws-sdk/deploy/wrapper/variables.tf deleted file mode 100644 index 8c23537e..00000000 --- a/python/sample-apps/aws-sdk/deploy/wrapper/variables.tf +++ /dev/null @@ -1,35 +0,0 @@ -variable "name" { - type = string - description = "Name of created function and API Gateway" - default = "hello-python" -} - -variable "collector_layer_arn" { - type = string - description = "ARN for the Lambda layer containing the OpenTelemetry collector extension" - // TODO(anuraaga): Add default when a public layer is published. -} - -variable "sdk_layer_arn" { - type = string - description = "ARN for the Lambda layer containing the OpenTelemetry Python Wrapper" - // TODO(anuraaga): Add default when a public layer is published. -} - -variable "tracing_mode" { - type = string - description = "Lambda function tracing mode" - default = "PassThrough" -} - -variable "architecture" { - type = string - description = "Lambda function architecture, valid values are arm64 or x86_64" - default = "x86_64" -} - -variable "runtime" { - type = string - description = "Python runtime version used for sample Lambda Function" - default = "python3.9" -} diff --git a/python/sample-apps/build.sh b/python/sample-apps/build.sh deleted file mode 100755 index e2be3fe0..00000000 --- a/python/sample-apps/build.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -set -e - -mkdir -p build/python -python3 -m pip install -r function/requirements.txt -t build/python -cp function/lambda_function.py build/python -cd build/python -zip -r ../function.zip ./* diff --git a/python/sample-apps/function/lambda_function.py b/python/sample-apps/function/lambda_function.py deleted file mode 100644 index a43ef3fb..00000000 --- a/python/sample-apps/function/lambda_function.py +++ /dev/null @@ -1,16 +0,0 @@ -import os -import json -import boto3 -import requests - -s3 = boto3.resource("s3") - -# lambda function -def lambda_handler(event, context): - - requests.get("http://httpbin.org/") - - for bucket in s3.buckets.all(): - print(bucket.name) - - return {"statusCode": 200, "body": json.dumps(os.environ.get("_X_AMZN_TRACE_ID"))} diff --git a/python/sample-apps/function/requirements.txt b/python/sample-apps/function/requirements.txt deleted file mode 100644 index cdb77fdf..00000000 --- a/python/sample-apps/function/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -requests -boto3 \ No newline at end of file diff --git a/python/sample-apps/run.sh b/python/sample-apps/run.sh deleted file mode 100755 index 53068868..00000000 --- a/python/sample-apps/run.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -euf -o pipefail - -cp -r ../src/otel . -cp ../src/run.sh layer.sh -./layer.sh "$@" -rm -rf .aws-sam -rm -rf otel -rm -r layer.sh diff --git a/python/sample-apps/template.yml b/python/sample-apps/template.yml deleted file mode 100644 index 4c0360fd..00000000 --- a/python/sample-apps/template.yml +++ /dev/null @@ -1,50 +0,0 @@ -AWSTemplateFormatVersion: '2010-09-09' -Transform: 'AWS::Serverless-2016-10-31' -Description: OpenTelemetry Python Lambda layer for Python -Resources: - api: - Type: AWS::Serverless::Api - Properties: - StageName: api - TracingEnabled: true - OpenApiVersion: 3.0.2 - function: - Type: AWS::Serverless::Function - Properties: - Handler: lambda_function.lambda_handler - Runtime: python3.9 - CodeUri: ./function - Description: Build OTel Python Lambda Sample from scratch - MemorySize: 512 - Timeout: 15 - Policies: - - AWSLambdaBasicExecutionRole - - AWSLambda_ReadOnlyAccess - - AWSXrayWriteOnlyAccess - - AmazonS3FullAccess - Environment: - Variables: - AWS_LAMBDA_EXEC_WRAPPER: /opt/python/otel-instrument - Tracing: Active - Layers: - - !Ref OTelLayer - Events: - getEndpoint: - Type: Api - Properties: - RestApiId: !Ref api - Path: / - Method: GET - OTelLayer: - Type: AWS::Serverless::LayerVersion - Properties: - LayerName: opentelemetry-python - Description: Opentelemetry Python layer - ContentUri: ./otel - CompatibleRuntimes: - - python3.8 - - python3.9 - - python3.10 - - python3.11 - Metadata: - BuildMethod: makefile diff --git a/dev/build-nodejs.sh b/scripts/build-nodejs.sh similarity index 96% rename from dev/build-nodejs.sh rename to scripts/build-nodejs.sh index e49df3d4..a5406a6f 100755 --- a/dev/build-nodejs.sh +++ b/scripts/build-nodejs.sh @@ -29,7 +29,7 @@ if [ ! -d "$OPENTELEMETRY_JS_PATH" ]; then git clone git@github.com:coralogix/opentelemetry-js.git "$OPENTELEMETRY_JS_PATH" -b coralogix-autoinstrumentation fi -"$ROOT_DIR/ci-scripts/build_nodejs_layer.sh" +"$ROOT_DIR/scripts/build_nodejs_layer.sh" # Useful for using the layer locally pushd "$ROOT_DIR/nodejs/packages/layer" >/dev/null diff --git a/ci-scripts/build_nodejs_layer.sh b/scripts/build_nodejs_layer.sh similarity index 100% rename from ci-scripts/build_nodejs_layer.sh rename to scripts/build_nodejs_layer.sh diff --git a/ci-scripts/check_size.sh b/scripts/check_size.sh similarity index 100% rename from ci-scripts/check_size.sh rename to scripts/check_size.sh diff --git a/dev/deploy-nodejs.sh b/scripts/deploy-nodejs.sh similarity index 96% rename from dev/deploy-nodejs.sh rename to scripts/deploy-nodejs.sh index 5013671f..a636b548 100755 --- a/dev/deploy-nodejs.sh +++ b/scripts/deploy-nodejs.sh @@ -18,7 +18,7 @@ if [ -z "${AWS_PROFILE:-}" ]; then echo "AWS_PROFILE not set, using default: $AWS_PROFILE" fi -"$ROOT_DIR/dev/build-nodejs.sh" +"$ROOT_DIR/scripts/build-nodejs.sh" output=$(aws lambda publish-layer-version \ --layer-name "$LAMBDA_LAYER_PREFIX-coralogix-opentelemetry-nodejs-wrapper-development" \ diff --git a/ci-scripts/publish-sandbox.sh b/scripts/publish-sandbox.sh similarity index 98% rename from ci-scripts/publish-sandbox.sh rename to scripts/publish-sandbox.sh index f2f57dec..00c4c24c 100755 --- a/ci-scripts/publish-sandbox.sh +++ b/scripts/publish-sandbox.sh @@ -40,7 +40,7 @@ echo "==> building collector ($ARCH)" make -C collector package-extend GOARCH="$ARCH" echo "==> building nodejs layer" -./ci-scripts/build_nodejs_layer.sh +./scripts/build_nodejs_layer.sh echo "==> merging zips" rm -rf build-sandbox && mkdir -p build-sandbox/out diff --git a/utils/sam/run.sh b/utils/sam/run.sh deleted file mode 100755 index 37c5b309..00000000 --- a/utils/sam/run.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - -set -euf -o pipefail - -echo_usage() { - echo "usage: Deploy Lambda layer/application by SAM" - echo " -r " - echo " -t " - echo " -b " - echo " -d " - echo " -n " - echo " -l " - echo " -s " -} - -is_sample() { - if [[ $(pwd) == *"sample"* ]]; then - echo 1 - else - echo 0 - fi -} - -main() { - echo "running..." - saved_args="$*" - template='template.yml' - build=false - deploy=false - layer=false - - stack=${OTEL_LAMBDA_STACK-"otel-stack"} - layerName=${OTEL_LAMBDA_LAYER-"otel-layer"} - - while getopts "hbdlr:t:s:n:" opt; do - case "${opt}" in - h) - echo_usage - exit 0 - ;; - b) - build=true - ;; - d) - deploy=true - ;; - n) - layerName="${OPTARG}" - ;; - l) - layer=true - ;; - r) - region="${OPTARG}" - ;; - t) - template="${OPTARG}" - ;; - s) - stack="${OPTARG}" - ;; - \?) - echo "Invalid option: -${OPTARG}" >&2 - exit 1 - ;; - :) - echo "Option -${OPTARG} requires an argument" >&2 - exit 1 - ;; - esac - done - - if [[ $deploy == true && $region == "" ]]; then - region=${AWS_REGION-$(aws configure get region)} - fi - - echo "Invoked with: ${saved_args}" - - if [[ $build == false && $deploy == false && $layer == false ]]; then - build=true - deploy=true - layer=true - fi - - if [[ $build == true ]]; then - echo "sam building..." - - echo "run.sh: Starting sam build." - sam build -u -t "$template" - zip -qr "$layerName".zip .aws-sam/build - fi - - if [[ $deploy == true ]]; then - sam deploy --stack-name "$stack" --region "$region" --capabilities CAPABILITY_NAMED_IAM --resolve-s3 --parameter-overrides LayerName="$layerName" - rm -rf otel/otel_collector - rm -f "$layerName".zip - fi - - if [[ $layer == true ]]; then - echo -e "\nOTel Lambda layer ARN:" - arn=$(aws lambda list-layer-versions --layer-name "$layerName" --region "$region" --query 'max_by(LayerVersions, &Version).LayerVersionArn') - echo "${arn//\"/}" - fi -} - -rm -rf .aws-sam - -main "$@" diff --git a/utils/terraform/api-gateway-proxy/main.tf b/utils/terraform/api-gateway-proxy/main.tf deleted file mode 100644 index 4bd8afbe..00000000 --- a/utils/terraform/api-gateway-proxy/main.tf +++ /dev/null @@ -1,67 +0,0 @@ -resource "aws_api_gateway_rest_api" "lambda_api_proxy" { - name = var.name -} - -resource "aws_api_gateway_resource" "lambda_api_proxy" { - rest_api_id = aws_api_gateway_rest_api.lambda_api_proxy.id - parent_id = aws_api_gateway_rest_api.lambda_api_proxy.root_resource_id - path_part = "{proxy+}" -} - -resource "aws_api_gateway_method" "lambda_api_proxy" { - rest_api_id = aws_api_gateway_rest_api.lambda_api_proxy.id - resource_id = aws_api_gateway_resource.lambda_api_proxy.id - http_method = "ANY" - authorization = "NONE" -} - -resource "aws_api_gateway_integration" "lambda_api" { - rest_api_id = aws_api_gateway_rest_api.lambda_api_proxy.id - resource_id = aws_api_gateway_method.lambda_api_proxy.resource_id - http_method = aws_api_gateway_method.lambda_api_proxy.http_method - - integration_http_method = "POST" - type = "AWS_PROXY" - uri = var.function_invoke_arn -} - -resource "aws_api_gateway_method" "lambda_api_proxy_root_nodejs" { - rest_api_id = aws_api_gateway_rest_api.lambda_api_proxy.id - resource_id = aws_api_gateway_rest_api.lambda_api_proxy.root_resource_id - http_method = "ANY" - authorization = "NONE" -} - -resource "aws_api_gateway_integration" "lambda_api_root_nodejs" { - rest_api_id = aws_api_gateway_rest_api.lambda_api_proxy.id - resource_id = aws_api_gateway_method.lambda_api_proxy_root_nodejs.resource_id - http_method = aws_api_gateway_method.lambda_api_proxy_root_nodejs.http_method - - integration_http_method = "POST" - type = "AWS_PROXY" - uri = var.function_invoke_arn -} - -resource "aws_api_gateway_deployment" "lambda_api_proxy" { - depends_on = [ - aws_api_gateway_integration.lambda_api, - aws_api_gateway_integration.lambda_api_root_nodejs, - ] - - rest_api_id = aws_api_gateway_rest_api.lambda_api_proxy.id -} - -resource "aws_api_gateway_stage" "test" { - stage_name = "default" - rest_api_id = aws_api_gateway_rest_api.lambda_api_proxy.id - deployment_id = aws_api_gateway_deployment.lambda_api_proxy.id - xray_tracing_enabled = var.enable_xray_tracing -} - -resource "aws_lambda_permission" "lambda_api_allow_gateway_nodejs" { - action = "lambda:InvokeFunction" - function_name = var.function_name - qualifier = var.function_qualifier - principal = "apigateway.amazonaws.com" - source_arn = "${aws_api_gateway_rest_api.lambda_api_proxy.execution_arn}/*/*" -} diff --git a/utils/terraform/api-gateway-proxy/outputs.tf b/utils/terraform/api-gateway-proxy/outputs.tf deleted file mode 100644 index 646329f4..00000000 --- a/utils/terraform/api-gateway-proxy/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "api_gateway_url" { - value = aws_api_gateway_stage.test.invoke_url -} diff --git a/utils/terraform/api-gateway-proxy/variables.tf b/utils/terraform/api-gateway-proxy/variables.tf deleted file mode 100644 index dbae1946..00000000 --- a/utils/terraform/api-gateway-proxy/variables.tf +++ /dev/null @@ -1,26 +0,0 @@ -variable "name" { - type = string - description = "Name of API gateway to create" -} - -variable "function_name" { - type = string - description = "Name of function to proxy to" -} - -variable "function_qualifier" { - type = string - default = null - description = "Qualifier of function to proxy to" -} - -variable "function_invoke_arn" { - type = string - description = "Invoke ARN of function to proxy to" -} - -variable "enable_xray_tracing" { - type = bool - description = "Whether to enable xray tracing of the API gateway" - default = false -}