diff --git a/.github/workflows/build-collector.yml b/.github/workflows/build-collector.yml deleted file mode 100644 index 7d11400ac5..0000000000 --- a/.github/workflows/build-collector.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: "Manual Build (Collector)" - -on: - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - architecture: [ amd64, arm64 ] - outputs: - COLLECTOR_VERSION: ${{ steps.save-collector-version.outputs.COLLECTOR_VERSION }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: '~1.21.9' - - name: Build Collector - run: make -C collector package GOARCH=${{ matrix.architecture }} - - name: Upload Collector Artifact - uses: actions/upload-artifact@v4 - with: - name: opentelemetry-collector-layer-${{ matrix.architecture }}.zip - path: ${{ github.workspace }}/collector/build/opentelemetry-collector-layer-${{ matrix.architecture }}.zip diff --git a/.github/workflows/layer-publish.yml b/.github/workflows/layer-publish.yml index 08730240cf..a015173280 100644 --- a/.github/workflows/layer-publish.yml +++ b/.github/workflows/layer-publish.yml @@ -32,7 +32,14 @@ on: description: 'Publish to which AWS region?' required: true type: string - + role-arn: + description: '(optional) AWS IAM Role ARN to be assumed for publishing layer. If no input is given, defaults to `PROD_LAMBDA_ROLE_ARN` secret.' + required: false + type: string + layer-version: + description: '(optional) Layer version to be used in the layer name. If no input is given, its value is tried to be extracted from the `GITHUB_REF_NAME` variable' + required: false + type: string permissions: id-token: write @@ -58,18 +65,24 @@ jobs: echo "ARCH=$ARCH" >> $GITHUB_ENV if [[ -n "${{ inputs.runtimes }}" ]]; then - RUNTIMES="--compatible-runtimes ${{ inputs.runtimes }}" + COMPATIBLE_RUNTIMES="--compatible-runtimes ${{ inputs.runtimes }}" fi - echo "RUNTIMES=$RUNTIMES" >> $GITHUB_ENV + echo "COMPATIBLE_RUNTIMES=$COMPATIBLE_RUNTIMES" >> $GITHUB_ENV if [[ "${{ inputs.release-group }}" != "prod" ]]; then LAYER_NAME=$LAYER_NAME-${{ inputs.release-group }} fi - LAYER_VERSION=$(echo "$GITHUB_REF_NAME" | sed -r 's/.*\/[^0-9\.]*//g') - LAYER_VERSION_CLEANED=$(echo "$LAYER_VERSION" | sed -r 's/\./_/g') + if [[ -n "${{ inputs.layer-version }}" ]]; then + LAYER_VERSION="${{ inputs.layer-version }}" + else + LAYER_VERSION=$(echo "$GITHUB_REF_NAME" | sed -r 's/.*\/[^0-9\.]*//g') + fi + LAYER_VERSION_CLEANED=$(echo "$LAYER_VERSION" | sed -r 's/\./_/g') - LAYER_NAME=$LAYER_NAME-$LAYER_VERSION_CLEANED + if [[ -n "$LAYER_VERSION_CLEANED" ]]; then + LAYER_NAME=$LAYER_NAME-$LAYER_VERSION_CLEANED + fi echo "LAYER_NAME=$LAYER_NAME" >> $GITHUB_ENV echo GITHUB_ENV: @@ -82,7 +95,7 @@ jobs: - uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: ${{ secrets.PROD_LAMBDA_ROLE_ARN }} + role-to-assume: ${{ inputs.role-arn || secrets.OTEL_LAMBDA_LAYER_PUBLISH_ROLE_ARN || secrets.PROD_LAMBDA_ROLE_ARN }} role-duration-seconds: 1200 aws-region: ${{ inputs.aws_region }} mask-aws-account-id: false @@ -93,7 +106,8 @@ jobs: aws lambda publish-layer-version \ --layer-name $LAYER_NAME \ --license-info "Apache 2.0" \ - --compatible-architectures $ARCH $RUNTIMES \ + --compatible-architectures $ARCH \ + $COMPATIBLE_RUNTIMES \ --zip-file fileb://${{ inputs.artifact-name }} \ --query 'LayerVersionArn' \ --output text diff --git a/.github/workflows/publish-layer-collector.yml b/.github/workflows/publish-layer-collector.yml new file mode 100644 index 0000000000..87feb27964 --- /dev/null +++ b/.github/workflows/publish-layer-collector.yml @@ -0,0 +1,131 @@ +name: "Publish Collector Lambda layer" + +on: + workflow_dispatch: + inputs: + architecture: + description: 'Architecture of the layer to be published' + required: true + type: choice + options: + - all + - amd64 + - arm64 + default: all + aws-region: + description: 'AWS Region(s) where layer will be published' + required: true + type: choice + options: + - all + - ap-northeast-1 + - ap-northeast-2 + - ap-south-1 + - ap-southeast-1 + - ap-southeast-2 + - ca-central-1 + - eu-central-1 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - sa-east-1 + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + default: all + role-arn: + description: 'AWS IAM Role ARN to be assumed for publishing layer' + required: false + type: string + layer-version: + description: 'Layer version to be appended into the layer name' + required: false + type: string + build-tags: + description: 'Build tags to customize collector build' + required: false + type: string + +jobs: + prepare-build-jobs: + runs-on: ubuntu-latest + outputs: + build_jobs: ${{ steps.prepare-build-jobs.outputs.build_jobs }} + steps: + - id: prepare-build-jobs + name: Prepare Build Jobs + run: | + architectures='' + if [ ${{ github.event.inputs.architecture }} == 'all' ]; then + architectures='["amd64", "arm64"]' + else + architectures='["${{ github.event.inputs.architecture }}"]' + fi + echo "build_jobs={"architecture": ${architectures}}" | tr -d '[:space:]' >> $GITHUB_OUTPUT + build-layer: + needs: prepare-build-jobs + runs-on: ubuntu-latest + strategy: + matrix: ${{ fromJSON(needs.prepare-build-jobs.outputs.build_jobs) }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '~1.21.9' + - name: Build Collector + run: | + if [[ -n "${{ inputs.build-tags }}" ]]; then + BUILDTAGS="${{ inputs.build-tags }}" + if [[ "$BUILDTAGS" != "lambdacomponents.custom"* ]]; then + BUILDTAGS="lambdacomponents.custom,$BUILDTAGS" + fi + fi + echo "Build tags: $BUILDTAGS" + make -C collector package GOARCH=${{ matrix.architecture }} BUILDTAGS=$BUILDTAGS + - name: Upload Collector Artifact + uses: actions/upload-artifact@v4 + with: + name: opentelemetry-collector-layer-${{ matrix.architecture }}.zip + path: ${{ github.workspace }}/collector/build/opentelemetry-collector-layer-${{ matrix.architecture }}.zip + prepare-release-jobs: + needs: build-layer + runs-on: ubuntu-latest + outputs: + release_jobs: ${{ steps.prepare-release-jobs.outputs.release_jobs }} + steps: + - id: prepare-release-jobs + name: Prepare Release Jobs + run: | + architectures='' + if [ ${{ github.event.inputs.architecture }} == 'all' ]; then + architectures='["amd64", "arm64"]' + else + architectures='["${{ github.event.inputs.architecture }}"]' + fi + aws_regions='' + if [ ${{ github.event.inputs.aws-region }} == 'all' ]; then + aws_regions='["ap-northeast-1", "ap-northeast-2", "ap-south-1", "ap-southeast-1", "ap-southeast-2", "ca-central-1", "eu-central-1", "eu-north-1", "eu-west-1", "eu-west-2", "eu-west-3", "sa-east-1", "us-east-1", "us-east-2", "us-west-1", "us-west-2"]' + else + aws_regions='["${{ github.event.inputs.aws-region }}"]' + fi + echo "release_jobs={"architecture": ${architectures}, "aws_region": ${aws_regions}}" | tr -d '[:space:]' >> $GITHUB_OUTPUT + release-layer: + uses: ./.github/workflows/layer-publish.yml + needs: prepare-release-jobs + strategy: + matrix: ${{ fromJSON(needs.prepare-release-jobs.outputs.release_jobs) }} + with: + artifact-name: opentelemetry-collector-layer-${{ matrix.architecture }}.zip + layer-name: opentelemetry-collector + architecture: ${{ matrix.architecture }} + runtimes: "nodejs16.x nodejs18.x nodejs20.x nodejs22.x java11 java17 java21 python3.8 python3.9 python3.10 python3.11 python3.12" + release-group: prod + aws_region: ${{ matrix.aws_region }} + role-arn: ${{ github.event.inputs.role-arn }} + component-version: 'NA' + layer-version: ${{ github.event.inputs.layer-version }} + secrets: inherit diff --git a/collector/README.md b/collector/README.md index 7d3b26fb9e..e64cd417fd 100644 --- a/collector/README.md +++ b/collector/README.md @@ -52,6 +52,40 @@ For example, if you want to add the extension `foo`, the file providing this ext You can provide your addition as a pull-request to this repository. Before doing so, please also read through the details of [Contributing](#contributing) to this project. +## Build and publish your own OpenTelemetry Collector Lambda layer + +To build and publish collector Lambda layer from your own fork into your own AWS account, +you can use the `Publish Collector Lambda Layer` workflow which can only be triggered manually. + + +To do that, first you need to +- Create Github's OIDC provider in your (or target) AWS account (for more details, you can check [here](https://github.com/aws-actions/configure-aws-credentials?oidc)) +- Create an AWS IAM Role in the AWS account to be assumed by the `Publish Collector Lambda Layer` workflow from your forked OpenTelemetry Lambda repository. + +To setup those, you can use (copy or load) the AWS CloudFormation template [here](../utils/aws-cloudformation/aws-cf-stack-for-layer-publish.yml). +Once AWS CloudFormation stack is created from the given template, +ARN of the created AWS IAM Role to be assumed will be shown as `RoleARN` in the output of the stack, so note it to be used later. + +After that, you can run the `Publish Collector Lambda Layer` workflow to build the Lambda collector and publish it to the target AWS account as Lambda layer: +- Specify the architecture of the collector Lambda layer to be published via the `Architecture of the layer to be published` input. + Available options are `all`, `amd64` and `arm64`. + The default value is `all` which builds and publishes layer for both of the `amd64` and `arm64` architectures. +- Specify the AWS region(s) where the collector Lambda layer will be published to via the `AWS Region(s) where layer will be published` input. + Available options are `all`, `ap-northeast-1`, `ap-northeast-2`, `ap-south-1`, `ap-southeast-1`, `ap-southeast-2`, `ca-central-1`, `eu-central-1`, `eu-north-1`, `eu-west-1`, `eu-west-2`, `eu-west-3`, `sa-east-1`, `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`. + The default value is `all` which publishes layer to all the defined AWS regions mentioned above. +- Specify the AWS IAM Role ARN to be assumed for publishing layer via the `AWS IAM Role ARN to be assumed for publishing layer` input. + This is the ARN of the AWS IAM Role you have taken from the `RoleARN` output variable of the created AWS CloudFormation stack above. + This input is **optional** and if not specified, AWS IAM Role ARN to be assumed is tried to be resolved from `OTEL_LAMBDA_LAYER_PUBLISH_ROLE_ARN` secret. + If it is still not able to resolved (neither this input is specified, nor `OTEL_LAMBDA_LAYER_PUBLISH_ROLE_ARN` secret is defined), + layer publish job will fail due to missing AWS credentials. +- Specify the layer version to be appended into layer name via the `Layer version to be appended into the layer name` input + to be used in the following format: `opentelemetry-lambda-collector-${architecture}-${layer-version}`. + This input is **optional** and if not specified, layer name is generated in the `opentelemetry-lambda-collector-${architecture}` format without layer version postfix. +- Specify the build tags to build the collector with a customized set of connectors/exporters/receivers/processors + via the `Build tags to customize collector build` input. + This input is **optional** and if not specified, collector is built with the default set of connectors/exporters/receivers/processors. + Check the [Build Tags](#build-tags) section for the details. + ## Installing To install the OpenTelemetry Collector Lambda layer to an existing Lambda function using the `aws` CLI: diff --git a/utils/aws-cloudformation/aws-cf-stack-for-layer-publish.yml b/utils/aws-cloudformation/aws-cf-stack-for-layer-publish.yml new file mode 100644 index 0000000000..10fe13d6d7 --- /dev/null +++ b/utils/aws-cloudformation/aws-cf-stack-for-layer-publish.yml @@ -0,0 +1,53 @@ +Parameters: + GitHubOrgName: + Description: Name of the GitHub organization/user + Type: String + RepositoryName: + Description: Name of the GitHub repository + Type: String + Default: "opentelemetry-lambda" + +Resources: + Role: + Type: AWS::IAM::Role + Properties: + RoleName: "github-otel-lambda-layer-publish-role" + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Action: sts:AssumeRoleWithWebIdentity + Principal: + Federated: !Ref GithubOIDC + Condition: + StringEquals: + token.actions.githubusercontent.com:aud: "sts.amazonaws.com" + StringLike: + token.actions.githubusercontent.com:sub: !Sub "repo:${GitHubOrgName}/${RepositoryName}:*" + Policies: + - PolicyName: "github-otel-lambda-layer-publish-policy" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - "lambda:GetLayer*" + - "lambda:ListLayer*" + - "lambda:AddLayer*" + - "lambda:PublishLayer*" + Resource: + - !Sub "arn:aws:lambda:*:${AWS::AccountId}:layer:opentelemetry-*" + - !Sub "arn:aws:lambda:*:${AWS::AccountId}:layer:opentelemetry-*:*" + + GithubOIDC: + Type: AWS::IAM::OIDCProvider + Properties: + Url: "https://token.actions.githubusercontent.com" + ClientIdList: + - "sts.amazonaws.com" + ThumbprintList: + - "ffffffffffffffffffffffffffffffffffffffff" + +Outputs: + RoleARN: + Description: "ARN of the AWS IAM role to be assumed by Github for the OpenTelemetry Layer publishing" + Value: !GetAtt Role.Arn