Skip to content

Commit e40c032

Browse files
KuznetsovRomanKuznetsovRoman
authored
Merge pull request #1268 from gemini-testing/TESTPLANE-1026.calibrate
fix: calibrate errors
2 parents 239bc20 + 431d8a5 commit e40c032

9 files changed

Lines changed: 120 additions & 28 deletions

File tree

‎src/browser/calibrator.ts‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import path from "path";
33
import looksSame from "looks-same";
44
import { CoreError } from "./core-error";
55
import { ExistingBrowser } from "./existing-browser";
6-
import type { Image } from "../image";
6+
import type { Image, RGB } from "../image";
77

88
const DIRECTION = { FORWARD: "forward", REVERSE: "reverse" } as const;
99

@@ -49,7 +49,11 @@ export class Calibrator {
4949

5050
const { innerWidth, pixelRatio } = features;
5151
const hasPixelRatio = Boolean(pixelRatio && pixelRatio > 1.0);
52-
const imageFeatures = await this._analyzeImage(image, { calculateColorLength: hasPixelRatio });
52+
const { width: imageWidth, height: imageHeight } = image.getSize();
53+
const searchColor = image.hasICCPChunk
54+
? await image.getRGB(Math.floor(imageWidth / 2), Math.floor(imageHeight / 2))
55+
: { R: 148, G: 250, B: 0 };
56+
const imageFeatures = await this._analyzeImage(image, { calculateColorLength: hasPixelRatio, searchColor });
5357

5458
if (!imageFeatures) {
5559
throw new CoreError(
@@ -70,9 +74,9 @@ export class Calibrator {
7074

7175
private async _analyzeImage(
7276
image: Image,
73-
params: { calculateColorLength?: boolean },
77+
params: { calculateColorLength?: boolean; searchColor: RGB },
7478
): Promise<ImageAnalysisResult | null> {
75-
const imageHeight = (await image.getSize()).height;
79+
const imageHeight = image.getSize().height;
7680

7781
for (let y = 0; y < imageHeight; y++) {
7882
const result = await analyzeRow(y, image, params);
@@ -88,9 +92,9 @@ export class Calibrator {
8892
async function analyzeRow(
8993
row: number,
9094
image: Image,
91-
params: { calculateColorLength?: boolean } = {},
95+
params: { calculateColorLength?: boolean; searchColor: RGB },
9296
): Promise<ImageAnalysisResult | null> {
93-
const markerStart = await findMarkerInRow(row, image, DIRECTION.FORWARD);
97+
const markerStart = await findMarkerInRow(row, image, DIRECTION.FORWARD, params.searchColor);
9498

9599
if (markerStart === -1) {
96100
return null;
@@ -102,15 +106,19 @@ async function analyzeRow(
102106
return result;
103107
}
104108

105-
const markerEnd = await findMarkerInRow(row, image, DIRECTION.REVERSE);
109+
const markerEnd = await findMarkerInRow(row, image, DIRECTION.REVERSE, params.searchColor);
106110
const colorLength = markerEnd - markerStart + 1;
107111

108112
return { ...result, colorLength };
109113
}
110114

111-
async function findMarkerInRow(row: number, image: Image, searchDirection: "forward" | "reverse"): Promise<number> {
112-
const imageWidth = (await image.getSize()).width;
113-
const searchColor = { R: 148, G: 250, B: 0 };
115+
async function findMarkerInRow(
116+
row: number,
117+
image: Image,
118+
searchDirection: "forward" | "reverse",
119+
searchColor: RGB,
120+
): Promise<number> {
121+
const imageWidth = image.getSize().width;
114122

115123
if (searchDirection === DIRECTION.REVERSE) {
116124
return searchReverse_();

‎src/browser/camera/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ export class Camera {
4545
const base64 = await this._takeScreenshot();
4646
const image = Image.fromBase64(base64);
4747

48-
const { width, height } = await image.getSize();
48+
const { width, height } = image.getSize();
4949
const imageArea: ImageArea = { left: 0, top: 0, width, height };
5050

5151
const calibratedArea = this._calibrateArea(imageArea);

‎src/browser/commands/assert-view/index.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ module.exports.default = browser => {
8888
const currImgInst = await screenShooter
8989
.capture(page, screenshoterOpts)
9090
.finally(() => browser.cleanupScreenshot(opts));
91-
const currSize = await currImgInst.getSize();
91+
const currSize = currImgInst.getSize();
9292
const currImg = { path: temp.path(Object.assign(tempOpts, { suffix: ".png" })), size: currSize };
9393

9494
const test = session.executionContext.ctx.currentTest;

‎src/browser/screen-shooter/viewport/index.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ module.exports = class Viewport {
3535
}
3636

3737
async handleImage(image, area = {}) {
38-
const { width, height } = await image.getSize();
38+
const { width, height } = image.getSize();
3939
_.defaults(area, { left: 0, top: 0, width, height });
4040
const capturedArea = this._transformToCaptureArea(area);
4141

@@ -57,7 +57,7 @@ module.exports = class Viewport {
5757

5858
async extendBy(physicalScrollHeight, newImage) {
5959
this._viewport.height += physicalScrollHeight;
60-
const { width, height } = await newImage.getSize();
60+
const { width, height } = newImage.getSize();
6161

6262
await this.handleImage(newImage, {
6363
left: 0,

‎src/image.ts‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,12 +80,39 @@ export const extractBase64PngSize = (base64EncodedPng: string): ImageSize => {
8080
};
8181
};
8282

83+
const hasICCPChunk = (buffer: Buffer): boolean => {
84+
const auxChunkSizeBytes = 12; // 4 bytes for length, 4 bytes for type, 4 bytes for crc
85+
const iCCPChunkType = Buffer.from("iCCP", "ascii").readUInt32BE(0);
86+
const IDATChunkType = Buffer.from("IDAT", "ascii").readUInt32BE(0);
87+
const PLTEChunkType = Buffer.from("PLTE", "ascii").readUInt32BE(0);
88+
89+
for (let nextChunkPointer = PNG_SIGNATURE.byteLength; nextChunkPointer <= buffer.length - auxChunkSizeBytes; ) {
90+
const chunkLength = buffer.readUInt32BE(nextChunkPointer);
91+
const chunkType = buffer.readUInt32BE(nextChunkPointer + 4);
92+
93+
if (chunkType === iCCPChunkType) {
94+
return true;
95+
}
96+
97+
// If the iCCP chunk appears, it must precede the first IDAT chunk, and it must also precede the PLTE chunk if present
98+
// https://libpng.org/pub/png/spec/1.2/PNG-Chunks.html#C.iCCP
99+
if (chunkType === IDATChunkType || chunkType === PLTEChunkType) {
100+
return false;
101+
}
102+
103+
nextChunkPointer += chunkLength + auxChunkSizeBytes;
104+
}
105+
106+
return false;
107+
};
108+
83109
export class Image {
84110
private _imgDataPromise: Promise<Buffer>;
85111
private _imgData: Buffer | null = null;
86112
private _width: number;
87113
private _height: number;
88114
private _composeImages: this[] = [];
115+
private _hasICCPChunk: boolean = false;
89116

90117
static create(buffer: Buffer): Image {
91118
return new this(buffer);
@@ -94,11 +121,16 @@ export class Image {
94121
constructor(buffer: Buffer) {
95122
this._width = buffer.readUInt32BE(PNG_WIDTH_OFFSET);
96123
this._height = buffer.readUInt32BE(PNG_HEIGHT_OFFSET);
124+
this._hasICCPChunk = hasICCPChunk(buffer);
97125
this._imgDataPromise = jsquashDecode(buffer).then(({ data }) => {
98126
return Buffer.from(data.buffer, data.byteOffset, data.byteLength);
99127
});
100128
}
101129

130+
public get hasICCPChunk(): boolean {
131+
return this._hasICCPChunk;
132+
}
133+
102134
async _getImgData(): Promise<Buffer> {
103135
if (this._imgData) {
104136
return this._imgData;
@@ -120,7 +152,7 @@ export class Image {
120152
}
121153
}
122154

123-
async getSize(): Promise<ImageSize> {
155+
getSize(): ImageSize {
124156
this._ensureImagesHaveSameWidth();
125157

126158
const height = this._composeImages.reduce((acc, img) => acc + img._height, this._height);

‎test/src/browser/calibrator.js‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,11 @@ describe("calibrator", () => {
2020
const setScreenshot = imageName => {
2121
const imgPath = path.join(__dirname, "..", "..", "fixtures", imageName);
2222
const imgData = fs.readFileSync(imgPath);
23+
const image = new Image(imgData);
2324

24-
browser.captureViewportImage.returns(Promise.resolve(new Image(imgData)));
25+
browser.captureViewportImage.returns(Promise.resolve(image));
26+
27+
return image;
2528
};
2629

2730
beforeEach(() => {
@@ -85,4 +88,53 @@ describe("calibrator", () => {
8588

8689
return assert.isRejected(calibrator.calibrate(browser), CoreError);
8790
});
91+
92+
describe("when image has iCCP chunk", () => {
93+
it("should use color at the center of the image to detect marker", async () => {
94+
const image = setScreenshot("calibrate.png");
95+
const { width, height } = image.getSize();
96+
const centerX = Math.floor(width / 2);
97+
const centerY = Math.floor(height / 2);
98+
const centerColor = { R: 50, G: 100, B: 150 };
99+
const markerLeft = 7;
100+
const markerTop = 4;
101+
102+
image._hasICCPChunk = true;
103+
104+
sinon.stub(image, "getRGB").callsFake(async (x, y) => {
105+
if (x === centerX && y === centerY) {
106+
return centerColor;
107+
}
108+
109+
if (x === markerLeft && y === markerTop) {
110+
return centerColor;
111+
}
112+
113+
return { R: 0, G: 0, B: 0 };
114+
});
115+
116+
const result = await calibrator.calibrate(browser);
117+
118+
assert.equal(result.top, markerTop);
119+
assert.equal(result.left, markerLeft);
120+
});
121+
});
122+
123+
describe("when image does not have iCCP chunk", () => {
124+
it("should use hardcoded green color as search color", async () => {
125+
const image = setScreenshot("calibrate.png");
126+
sinon.stub(image, "getRGB").callsFake(async (x, y) => {
127+
if (x === 3 && y === 5) {
128+
return { R: 148, G: 250, B: 0 };
129+
}
130+
131+
return { R: 0, G: 0, B: 0 };
132+
});
133+
134+
const result = await calibrator.calibrate(browser);
135+
136+
assert.equal(result.top, 5);
137+
assert.equal(result.left, 3);
138+
});
139+
});
88140
});

‎test/src/browser/camera/index.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ describe("browser/camera", () => {
1818
}).Camera;
1919

2020
image = sinon.createStubInstance(Image);
21-
image.getSize.resolves({ width: 100500, height: 500100 });
21+
image.getSize.returns({ width: 100500, height: 500100 });
2222
image.crop.resolves();
2323

2424
sandbox.stub(Image, "fromBase64").returns(image);
@@ -40,7 +40,7 @@ describe("browser/camera", () => {
4040
describe("calibration", () => {
4141
it("should apply calibration on taken screenshot", async () => {
4242
const camera = Camera.create(null, sinon.stub().resolves());
43-
image.getSize.resolves({ width: 10, height: 10 });
43+
image.getSize.returns({ width: 10, height: 10 });
4444

4545
camera.calibrate({ top: 6, left: 4 });
4646
await camera.captureViewportImage();

‎test/src/browser/screen-shooter/viewport/index.js‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ describe("Viewport", () => {
2323

2424
beforeEach(() => {
2525
image = sandbox.createStubInstance(Image);
26-
image.getSize.resolves({ width: 100500, height: 500100 });
26+
image.getSize.returns({ width: 100500, height: 500100 });
2727
});
2828

2929
afterEach(() => sandbox.restore());
@@ -176,7 +176,7 @@ describe("Viewport", () => {
176176
});
177177

178178
describe("should crop to captureArea", () => {
179-
beforeEach(() => image.getSize.resolves({ width: 7, height: 10 }));
179+
beforeEach(() => image.getSize.returns({ width: 7, height: 10 }));
180180

181181
it("with default area", async () => {
182182
const vieport = createViewport({
@@ -270,7 +270,7 @@ describe("Viewport", () => {
270270
newImage = sinon.createStubInstance(Image);
271271

272272
newImage.crop.resolves();
273-
newImage.getSize.resolves({});
273+
newImage.getSize.returns({});
274274
});
275275

276276
it("should increase viewport height value by scroll height", async () => {
@@ -285,7 +285,7 @@ describe("Viewport", () => {
285285
});
286286

287287
it("should crop new image by passed scroll height", async () => {
288-
newImage.getSize.resolves({ height: 4, width: 2 });
288+
newImage.getSize.returns({ height: 4, width: 2 });
289289
const viewport = createViewport({
290290
captureArea: { left: 0, top: 0, width: 4, height: 20 },
291291
viewport: { left: 0, top: 0, width: 4, height: 8 },

‎test/src/image.js‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -182,33 +182,33 @@ describe("Image", () => {
182182
});
183183

184184
describe("getSize", () => {
185-
it("should return size of single image", async () => {
185+
it("should return size of single image", () => {
186186
const buffer = createMockPngBuffer(100, 50);
187187
const image = new Image(buffer);
188188

189-
const size = await image.getSize();
189+
const size = image.getSize();
190190

191191
assert.deepEqual(size, { width: 100, height: 50 });
192192
});
193193

194-
it("should return combined height for composed images", async () => {
194+
it("should return combined height for composed images", () => {
195195
const buffer = createMockPngBuffer(100, 50);
196196
const image = new Image(buffer);
197197
const attachedImage1 = new Image(createMockPngBuffer(100, 30));
198198
const attachedImage2 = new Image(createMockPngBuffer(100, 20));
199199
image._composeImages = [attachedImage1, attachedImage2];
200200

201-
const size = await image.getSize();
201+
const size = image.getSize();
202202

203203
assert.deepEqual(size, { width: 100, height: 100 }); // 50 + 30 + 20
204204
});
205205

206-
it("should ensure images have same width before calculating size", async () => {
206+
it("should ensure images have same width before calculating size", () => {
207207
const buffer = createMockPngBuffer(100, 50);
208208
const image = new Image(buffer);
209209
sandbox.spy(image, "_ensureImagesHaveSameWidth");
210210

211-
await image.getSize();
211+
image.getSize();
212212

213213
assert.calledOnce(image._ensureImagesHaveSameWidth);
214214
});

0 commit comments

Comments
 (0)