abcoder/.github/workflows/regression.yml at 899b84649bfc35a0676ce9be17ea2838e7450dfa · cloudwego/abcoder · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
name: Regression Test

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

jobs:
  run_all_tests:
    runs-on: ubuntu-latest
    #if: "!contains(github.event.pull_request.title, '[NO-REGRESSION-TEST]')"
    env:
      LANGS: "go rust python java typescript"
    steps:
      - uses: actions/checkout@v4
        with: { path: 'pr_repo' }
      - uses: actions/checkout@v4
        with: { path: 'main_repo', ref: 'main' }

      - uses: actions/setup-go@v5
        with:
          go-version: '1.22'
          cache-dependency-path: main_repo/go.sum
      - uses: dtolnay/rust-toolchain@stable
        with:
          toolchain: stable
          components: rust-analyzer
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - uses: actions/setup-java@v4
        with:
          java-version: '21'
          distribution: 'temurin'
      - uses: actions/setup-node@v4
        with:
          node-version: '22'

      - name: Compile both binaries
        run: |
          (cd main_repo && go build -o ../abcoder_old)
          (cd pr_repo && go build -o ../abcoder_new)

      # Uses the new evaluation script.
      - name: Install evaluation dependencies
        run: |
          pip install -r ./pr_repo/script/requirements.txt

      - name: Install dependencies for OLD
        run: |
          OUTDIR=out_old ABCEXE=./abcoder_old ./pr_repo/script/run_testdata.sh first
          echo "JDTLS_ROOT_PATH=$(realpath ./pr_repo/lang/java/lsp/jdtls/jdt-language-server-*)" >> $GITHUB_ENV

      - name: Run OLD abcoder
        run:
          OUTDIR=out_old ABCEXE=./abcoder_old ./pr_repo/script/run_testdata.sh all

      # Whether to reset dependencies:
      # [n] Go:         builtin parser, do not reset
      # [n] Rust:       assume rls to be same version
      # [n] Python:     assume pylsp to be same version
      # [n] Java:       assume jdtls to be same version
      # [y] TypeScript: need to reset installed parser
      - name: Reset dependencies
        run: |
          npm uninstall -g abcoder-ts-parser

      - name: Install dependencies for NEW
        run: |
          OUTDIR=out_new ABCEXE=./abcoder_new ./pr_repo/script/run_testdata.sh first

      - name: Run NEW abcoder
        run:
          OUTDIR=out_new ABCEXE=./abcoder_new ./pr_repo/script/run_testdata.sh all

      - name: Upload output directories
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: regression-outputs
          path: |
            out_old
            out_new
          retention-days: 3

      - name: Compare outputs and check for regression
        run: ./pr_repo/script/diffjson.py out_old out_new