Merge pull request #38 from sprahasingh/fix/github-actions-temporary-ssh

fix: secure GitHub Actions deployment with temporary SSH access

Author: sprahasingh

.github/workflows/production.yml

@@ -4,12 +4,14 @@ on:
   push:
     branches: [main]
   pull_request:
+  schedule:
+    - cron: "*/15 * * * *"
   workflow_dispatch:
     inputs:
       operation:
         description: Production action
         type: choice
-        options: [deploy, deploy_existing, rollback]
+        options: [deploy, deploy_existing, rollback, recover_ssh]
         default: deploy
         required: true
       publication_run_id:
@@ -38,7 +40,9 @@ concurrency:
 
 jobs:
   test:
-    if: github.event_name != 'workflow_dispatch' || inputs.operation == 'deploy' || inputs.operation == 'deploy_existing'
+    if: >-
+      github.event_name == 'push' || github.event_name == 'pull_request' ||
+      (github.event_name == 'workflow_dispatch' && (inputs.operation == 'deploy' || inputs.operation == 'deploy_existing'))
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -96,8 +100,10 @@ jobs:
           --volume "$GITHUB_WORKSPACE/tests:/tmp/codelens-tests:ro"
           --volume "$GITHUB_WORKSPACE/scripts:/tmp/scripts:ro"
           --volume "$GITHUB_WORKSPACE/.github/workflows/production.yml:/tmp/production.yml:ro"
+          --volume "$GITHUB_WORKSPACE/docs/ghcr-deployment.md:/tmp/ghcr-deployment.md:ro"
           --volume "$GITHUB_WORKSPACE/pytest.ini:/tmp/pytest.ini:ro"
           --env CODELENS_WORKFLOW_FILE=/tmp/production.yml
+          --env CODELENS_DEPLOYMENT_DOC=/tmp/ghcr-deployment.md
           --env PYTHONPATH=/tmp
           --env DEBUG=false
           --env DATABASE_URL=postgresql+asyncpg://test:test@127.0.0.1/test
@@ -111,7 +117,10 @@ jobs:
 
   publish:
     needs: test
-    if: github.ref == 'refs/heads/main' && (github.event_name != 'workflow_dispatch' || inputs.operation == 'deploy')
+    if: >-
+      github.ref == 'refs/heads/main' &&
+      ((github.event_name == 'push') ||
+      (github.event_name == 'workflow_dispatch' && inputs.operation == 'deploy'))
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -150,8 +159,10 @@ jobs:
           --volume "$GITHUB_WORKSPACE/tests:/tmp/codelens-tests:ro"
           --volume "$GITHUB_WORKSPACE/scripts:/tmp/scripts:ro"
           --volume "$GITHUB_WORKSPACE/.github/workflows/production.yml:/tmp/production.yml:ro"
+          --volume "$GITHUB_WORKSPACE/docs/ghcr-deployment.md:/tmp/ghcr-deployment.md:ro"
           --volume "$GITHUB_WORKSPACE/pytest.ini:/tmp/pytest.ini:ro"
           --env CODELENS_WORKFLOW_FILE=/tmp/production.yml
+          --env CODELENS_DEPLOYMENT_DOC=/tmp/ghcr-deployment.md
           --env PYTHONPATH=/tmp
           --env DEBUG=false
           --env DATABASE_URL=postgresql+asyncpg://test:test@127.0.0.1/test
@@ -228,14 +239,31 @@ jobs:
       (github.event_name == 'workflow_dispatch' && inputs.operation == 'deploy_existing' && needs.verify_existing.result == 'success') ||
       (github.event_name == 'push' && needs.publish.result == 'success' && vars.CODELENS_AUTO_DEPLOY == 'true'))
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     environment: production
     permissions:
       contents: read
       packages: read
+      id-token: write
     steps:
       - uses: actions/checkout@v7.0.1
         with:
           persist-credentials: false
+      - name: Assume the scoped AWS deployment role
+        uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0
+        with:
+          role-to-assume: ${{ vars.CODELENS_AWS_ROLE_ARN }}
+          aws-region: eu-north-1
+          audience: sts.amazonaws.com
+          role-session-name: codelens-${{ github.run_id }}-${{ github.run_attempt }}
+          mask-aws-account-id: true
+      - name: Authorize this runner's temporary SSH rule
+        id: ssh_access
+        env:
+          RUN_ID: ${{ github.run_id }}
+          RUN_ATTEMPT: ${{ github.run_attempt }}
+        shell: bash
+        run: python3 scripts/manage_temporary_ssh.py authorize --run-id "$RUN_ID" --attempt "$RUN_ATTEMPT"
       - name: Configure verified SSH host key
         env:
           EC2_SSH_PRIVATE_KEY: ${{ secrets.EC2_SSH_PRIVATE_KEY }}
@@ -304,17 +332,42 @@ jobs:
           host="$EC2_USER@$EC2_HOST"
           ssh_args=(-i "$SSH_DIR/id_ed25519" -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$SSH_DIR/known_hosts")
           ssh "${ssh_args[@]}" "$host" "rm -f '/tmp/codelens-deploy-${RUN_ID}-${RUN_ATTEMPT}.sh' '/tmp/codelens-compose-${RUN_ID}-${RUN_ATTEMPT}.yml' '/tmp/codelens-ghcr-token-${RUN_ID}-${RUN_ATTEMPT}'" || true
+      - name: Revoke this run's temporary SSH rule
+        if: always() && steps.ssh_access.outputs.ssh_rule_id != ''
+        env:
+          SSH_RULE_ID: ${{ steps.ssh_access.outputs.ssh_rule_id }}
+          RUN_ID: ${{ github.run_id }}
+          RUN_ATTEMPT: ${{ github.run_attempt }}
+        shell: bash
+        run: python3 scripts/manage_temporary_ssh.py revoke --rule-id "$SSH_RULE_ID" --run-id "$RUN_ID" --attempt "$RUN_ATTEMPT"
 
   rollback:
     if: github.event_name == 'workflow_dispatch' && inputs.operation == 'rollback' && github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     environment: production
     permissions:
       contents: read
+      id-token: write
     steps:
       - uses: actions/checkout@v7.0.1
         with:
           persist-credentials: false
+      - name: Assume the scoped AWS deployment role
+        uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0
+        with:
+          role-to-assume: ${{ vars.CODELENS_AWS_ROLE_ARN }}
+          aws-region: eu-north-1
+          audience: sts.amazonaws.com
+          role-session-name: codelens-rollback-${{ github.run_id }}-${{ github.run_attempt }}
+          mask-aws-account-id: true
+      - name: Authorize this runner's temporary SSH rule
+        id: ssh_access
+        env:
+          RUN_ID: ${{ github.run_id }}
+          RUN_ATTEMPT: ${{ github.run_attempt }}
+        shell: bash
+        run: python3 scripts/manage_temporary_ssh.py authorize --run-id "$RUN_ID" --attempt "$RUN_ATTEMPT"
       - name: Configure verified SSH host key
         env:
           EC2_SSH_PRIVATE_KEY: ${{ secrets.EC2_SSH_PRIVATE_KEY }}
@@ -359,3 +412,38 @@ jobs:
           host="$EC2_USER@$EC2_HOST"
           ssh_args=(-i "$SSH_DIR/id_ed25519" -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$SSH_DIR/known_hosts")
           ssh "${ssh_args[@]}" "$host" "rm -f '/tmp/codelens-rollback-${RUN_ID}-${RUN_ATTEMPT}.sh' '/tmp/codelens-rollback-${RUN_ID}-${RUN_ATTEMPT}.yml'" || true
+      - name: Revoke this run's temporary SSH rule
+        if: always() && steps.ssh_access.outputs.ssh_rule_id != ''
+        env:
+          SSH_RULE_ID: ${{ steps.ssh_access.outputs.ssh_rule_id }}
+          RUN_ID: ${{ github.run_id }}
+          RUN_ATTEMPT: ${{ github.run_attempt }}
+        shell: bash
+        run: python3 scripts/manage_temporary_ssh.py revoke --rule-id "$SSH_RULE_ID" --run-id "$RUN_ID" --attempt "$RUN_ATTEMPT"
+
+  recover_ssh:
+    if: >-
+      github.ref == 'refs/heads/main' &&
+      ((github.event_name == 'schedule') ||
+      (github.event_name == 'workflow_dispatch' && inputs.operation == 'recover_ssh'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    environment: production-ssh-recovery
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v7.0.1
+        with:
+          persist-credentials: false
+      - name: Assume the scoped AWS cleanup role
+        uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0
+        with:
+          role-to-assume: ${{ vars.CODELENS_AWS_ROLE_ARN }}
+          aws-region: eu-north-1
+          audience: sts.amazonaws.com
+          role-session-name: codelens-ssh-recovery-${{ github.run_id }}-${{ github.run_attempt }}
+          mask-aws-account-id: true
+      - name: Remove expired CodeLens runner SSH rules
+        shell: bash
+        run: python3 scripts/manage_temporary_ssh.py recover-expired

docs/ghcr-deployment.md

@@ -115,3 +115,213 @@ cancels an in-progress deployment.
 GitHub Actions and GHCR have account-specific included usage and storage limits.
 Monitor the account's current quotas; immutable commit tags and rollback images
 are intentionally retained rather than automatically deleted.
+
+## Temporary GitHub Actions SSH access with AWS OIDC
+
+The EC2 security group sg-05c1462a133b8d00f is managed by a dedicated
+GitHub Actions role. For deploy and rollback runs, the workflow obtains a
+short-lived AWS session through GitHub OIDC, reads the current runner IPv4 from
+https://checkip.amazonaws.com/ over verified HTTPS, validates that the
+response is exactly one globally routable IPv4 address, and adds one TCP/22
+/32 ingress rule. The current fixed rule TTL is two hours. The helper stores
+the AWS-returned sgr-... rule ID as a job output and passes that exact ID to
+the cleanup command. It never searches by or revokes the Mac rule
+sgr-070a3d2767818f890; the existing Mac, HTTP, and HTTPS rules are otherwise
+left alone.
+
+The revoke helper first describes the requested rule in the fixed security
+group and checks its group, rule ID, direction, TCP/22 ports, public /32 source,
+management tags, description, and this workflow run identity. It then calls
+revoke-security-group-ingress with that one AWS rule ID and verifies the rule
+is absent. Errors are reduced to fixed messages. A failed revoke fails the job.
+The workflow's always() cleanup step handles normal failures and cancellation.
+A scheduled job runs every 15 minutes in the separate
+production-ssh-recovery GitHub environment and removes only expired rules with
+the exact CodeLens owner/run/expiry tags and matching run description. The
+recovery environment must be restricted
+to main, have no secrets, and have no reviewer gate so cleanup can run
+unattended. It receives the same narrowly scoped AWS role. A manual recover_ssh
+dispatch from main is also available.
+
+### GitHub setup
+
+1. Keep the existing protected production environment and its required
+   reviewers. Its deployment branches must remain restricted to main.
+2. Create production-ssh-recovery. Restrict its deployment branches to main;
+   add no secrets and no required reviewers. It is used only by the 15-minute
+   stale-rule reaper and manual recover_ssh operation.
+3. Add repository variable CODELENS_AWS_ROLE_ARN with the ARN of the IAM role
+   described below. It is an ARN, not a credential. Do not add AWS access keys.
+4. Keep EC2_HOST, EC2_USER, EC2_SSH_PRIVATE_KEY, and EC2_KNOWN_HOSTS secrets
+   only in production. No EC2 SSH secret is used by the recovery job.
+5. The IAM OIDC provider URL is
+   https://token.actions.githubusercontent.com, with audience sts.amazonaws.com.
+   Add the provider only if it does not already exist.
+
+The verified GitHub repository OIDC settings are `use_default: true`,
+`use_immutable_subject: true`, and immutable subject prefix
+`repo@99238925/CodeLens@1334470875`. No custom include-claim-key list is
+configured. With an environment, the exact subject values are
+`repo:sprahasingh@99238925/CodeLens@1334470875:environment:production` and
+`repo:sprahasingh@99238925/CodeLens@1334470875:environment:production-ssh-recovery`.
+GitHub documents this immutable `repo:OWNER@OWNER-ID/REPO@REPO-ID` syntax and
+the `:environment:ENVIRONMENT` suffix for environment jobs. The trust policy
+below accepts only those two subjects, the expected audience, repository and
+owner IDs, and `main`.
+
+### IAM role trust policy
+
+Create the IAM role `CodeLensGitHubActionsSSH` and use this trust policy.
+The account and GitHub IDs below are the verified values. The OIDC provider
+must exist in AWS account `510155707736`, region `eu-north-1`.
+
+    {
+      "Version": "2012-10-17",
+      "Statement": [
+        {
+          "Effect": "Allow",
+          "Principal": {
+            "Federated": "arn:aws:iam::510155707736:oidc-provider/token.actions.githubusercontent.com"
+          },
+          "Action": "sts:AssumeRoleWithWebIdentity",
+          "Condition": {
+            "StringEquals": {
+              "token.actions.githubusercontent.com:aud": "sts.amazonaws.com",
+              "token.actions.githubusercontent.com:repository_owner_id": "99238925",
+              "token.actions.githubusercontent.com:repository_id": "1334470875",
+              "token.actions.githubusercontent.com:ref": "refs/heads/main",
+              "token.actions.githubusercontent.com:environment": [
+                "production",
+                "production-ssh-recovery"
+              ],
+              "token.actions.githubusercontent.com:sub": [
+                "repo:sprahasingh@99238925/CodeLens@1334470875:environment:production",
+                "repo:sprahasingh@99238925/CodeLens@1334470875:environment:production-ssh-recovery"
+              ]
+            }
+          }
+        }
+      ]
+    }
+
+### IAM permissions policy
+
+Attach only this inline policy to that role. The helper authorizes rules with
+`TagSpecifications` on the `AuthorizeSecurityGroupIngress` API call; it does
+not make a separate `CreateTags` API call. EC2 performs dependent authorization
+for `ec2:CreateTags` when tags are supplied during rule creation. This policy
+allows that tag-on-create path only, limits ingress authorization and
+revocation to the CodeLens security group, and permits read-only rule
+inspection in `eu-north-1`. It grants no instance, key, network-interface, or
+other security-group management permissions.
+
+    {
+      "Version": "2012-10-17",
+      "Statement": [
+        {
+          "Sid": "DescribeSecurityGroupRules",
+          "Effect": "Allow",
+          "Action": "ec2:DescribeSecurityGroupRules",
+          "Resource": "*",
+          "Condition": {
+            "StringEquals": {"ec2:Region": "eu-north-1"}
+          }
+        },
+        {
+          "Sid": "AuthorizeIngressOnCodeLensGroup",
+          "Effect": "Allow",
+          "Action": "ec2:AuthorizeSecurityGroupIngress",
+          "Resource": "arn:aws:ec2:eu-north-1:510155707736:security-group/sg-05c1462a133b8d00f",
+          "Condition": {
+            "StringEquals": {
+              "ec2:SecurityGroupID": "sg-05c1462a133b8d00f",
+              "ec2:Region": "eu-north-1"
+            }
+          }
+        },
+        {
+          "Sid": "AuthorizeTaggedRuleCreation",
+          "Effect": "Allow",
+          "Action": "ec2:AuthorizeSecurityGroupIngress",
+          "Resource": "arn:aws:ec2:eu-north-1:510155707736:security-group-rule/*",
+          "Condition": {
+            "StringEquals": {
+              "aws:RequestTag/CodeLensManagedBy": "CodeLensGitHubActions",
+              "ec2:Region": "eu-north-1"
+            },
+            "ForAllValues:StringEquals": {
+              "aws:TagKeys": ["CodeLensManagedBy", "CodeLensRun", "CodeLensExpiresAt"]
+            },
+            "Null": {
+              "aws:RequestTag/CodeLensRun": "false",
+              "aws:RequestTag/CodeLensExpiresAt": "false"
+            }
+          }
+        },
+        {
+          "Sid": "TagOnlyRulesCreatedByIngressAuthorization",
+          "Effect": "Allow",
+          "Action": "ec2:CreateTags",
+          "Resource": "arn:aws:ec2:eu-north-1:510155707736:security-group-rule/*",
+          "Condition": {
+            "StringEquals": {
+              "ec2:CreateAction": "AuthorizeSecurityGroupIngress",
+              "aws:RequestTag/CodeLensManagedBy": "CodeLensGitHubActions",
+              "ec2:Region": "eu-north-1"
+            },
+            "ForAllValues:StringEquals": {
+              "aws:TagKeys": ["CodeLensManagedBy", "CodeLensRun", "CodeLensExpiresAt"]
+            },
+            "Null": {
+              "aws:RequestTag/CodeLensRun": "false",
+              "aws:RequestTag/CodeLensExpiresAt": "false"
+            }
+          }
+        },
+        {
+          "Sid": "RevokeIngressOnlyOnCodeLensGroup",
+          "Effect": "Allow",
+          "Action": "ec2:RevokeSecurityGroupIngress",
+          "Resource": "arn:aws:ec2:eu-north-1:510155707736:security-group/sg-05c1462a133b8d00f",
+          "Condition": {
+            "StringEquals": {
+              "ec2:SecurityGroupID": "sg-05c1462a133b8d00f",
+              "ec2:Region": "eu-north-1"
+            }
+          }
+        }
+      ]
+    }
+
+AWS authorizes ingress authorization and revocation at the security-group
+level; it does not let IAM constrain a request to TCP/22, a /32, or one specific
+rule ID. `ec2:SecurityGroupID` is applied only to the security-group resource
+authorization check; AWS lists request-tag keys for the separate
+security-group-rule resource check. The helper enforces the port, /32, and
+rule-ownership checks, revokes by exact rule ID, and explicitly protects both
+the known Mac rule ID and its current /32 source. A compromised workflow role
+could still add another ingress rule or revoke another rule in this one group;
+isolating runner SSH rules in a dedicated security group would remove that
+IAM-level limitation but requires a separately approved infrastructure
+change. `DescribeSecurityGroupRules` has no resource-level ARN, so its read
+permission uses `Resource: "*"`; the region condition is the narrowest IAM
+scope available, while the helper filters requests to the CodeLens group.
+
+### Retry and recovery
+
+After configuring the role, provider, repository variable, and recovery
+environment, rerun Actions → Test, publish, and deploy CodeLens → Run workflow
+from main with operation=deploy_existing and the same verified publication
+run ID, source commit, and image digest. Do not select deploy; that mode
+publishes a new image. The workflow still runs tests, validates the previous
+successful publication, waits for production approval, then applies the
+existing Celery idle gate and deployment-helper rollback protections.
+
+On the next scheduled recovery run, expired managed rules are removed. If
+cleanup fails, inspect the sanitized recovery result and manually dispatch
+recover_ssh from main; do not remove a rule by CIDR or description. If an
+expired managed rule cannot pass validation, the reaper fails closed and
+requires manual inspection of that exact tagged rule. No automated process
+can guarantee cleanup during a prolonged GitHub Actions outage; the two-hour
+expiry marker and scheduled reaper bound expected exposure when Actions is
+available.