@@ -15,10 +15,24 @@ jobs:
1515 env :
1616 LANGS : " go rust python java typescript"
1717 steps :
18+ - name : Checkout pull request code
19+ uses : actions/checkout@v4
20+ with :
21+ path : ' pr_repo'
22+
23+ - name : Checkout main branch code
24+ uses : actions/checkout@v4
25+ with :
26+ ref : ' main'
27+ path : ' main_repo'
28+
1829 - name : Setup Go environment
1930 uses : actions/setup-go@v5
2031 with :
2132 go-version : ' 1.22'
33+ cache-dependency-path : |
34+ main_repo/go.sum
35+ pr_repo/go.sum
2236
2337 - name : Setup Rust toolchain
2438 uses : dtolnay/rust-toolchain@stable
4256 with :
4357 node-version : ' 22'
4458
45- - name : Checkout pull request code
46- uses : actions/checkout@v4
47- with :
48- path : ' pr_repo'
49-
50- - name : Checkout main branch code
51- uses : actions/checkout@v4
52- with :
53- ref : ' main'
54- path : ' main_repo'
55-
5659 - name : Compile both binaries
5760 run : |
5861 (cd main_repo && go build -o ../abcoder_old)
@@ -61,25 +64,32 @@ jobs:
6164 - name : Install evaluation dependencies
6265 run : pip install -r ./pr_repo/script/requirements.txt
6366
64- - name : Install LSPs
67+ # #############################################################################
68+ - name : Install dependencies for old
6569 run : |
66- OUTDIR=out_new ABCEXE=./abcoder_new ./pr_repo/script/run_testdata.sh first
67- # use the same JDTLS for consistency and to avoid wasting time installing a duplicate JDTLS
68- echo "JDTLS_ROOT_PATH=$(realpath ./pr_repo/lang/java/lsp/jdtls/jdt-language-server-*)" >> $GITHUB_ENV
70+ # HACK: auto installation uses the published version, not our local version
71+ (cd ./main_repo/ts-parser && npm install && npm install -g .)
72+ OUTDIR=out_old ABCEXE=./abcoder_old ./main_repo/script/run_testdata.sh first
73+ # avoid wasting time install a new jdtls
74+ echo "JDTLS_ROOT_PATH=$(realpath ./main_repo/lang/java/lsp/jdtls/jdt-language-server-*)" >> $GITHUB_ENV
6975
7076 - name : Run OLD abcoder
7177 run :
72- OUTDIR=out_old ABCEXE=./abcoder_old ./pr_repo/script/run_testdata.sh all
78+ OUTDIR=out_old ABCEXE=./abcoder_old ./main_repo/script/run_testdata.sh all
79+
80+ - name : Reset dependencies
81+ run : |
82+ npm uninstall -g abcoder-ts-parser
83+
84+ - name : Install dependencies for new
85+ run : |
86+ (cd ./pr_repo/ts-parser && npm install && npm install -g .)
87+ OUTDIR=out_new ABCEXE=./abcoder_new ./pr_repo/script/run_testdata.sh first
7388
7489 - name : Run NEW abcoder
7590 run :
7691 OUTDIR=out_new ABCEXE=./abcoder_new ./pr_repo/script/run_testdata.sh all
7792
78- - name : Compare outputs and check for regression
79- id : diff_check
80- run : ./pr_repo/script/diffjson.py out_old out_new || echo "failed=true" >> $GITHUB_OUTPUT
81- continue-on-error : true
82-
8393 - name : Upload output directories
8494 uses : actions/upload-artifact@v4
8595 if : always()
90100 out_new
91101 retention-days : 3
92102
93- - name : Status check
94- if : steps.diff_check.outputs.failed == 'true'
95- run : exit 1
103+ - name : Compare outputs and check for regression
104+ run : ./pr_repo/script/diffjson.py out_old out_new
0 commit comments