diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 107457bf..236c55c8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -151,6 +151,13 @@ jobs: test_commands: | compile simple-batch-test-package.json + - example: canary-scheduled-batch + path: test-jobs/scheduled-batch-test + tag: canary + test_commands: | + compile scheduled-batch-package.json + compile simple-scheduled-batch-package.json + - example: udf-jbang path: user-defined-function/jbang tag: udf-jbang diff --git a/test-jobs/scheduled-batch-test/connectors/addresses.jsonl b/test-jobs/scheduled-batch-test/connectors/addresses.jsonl new file mode 100644 index 00000000..c2e12e2b --- /dev/null +++ b/test-jobs/scheduled-batch-test/connectors/addresses.jsonl @@ -0,0 +1,100 @@ +{"id":1,"name":"John Doe","street":"123 Maple Street","city":"Springfield","zip":"12345","country":"USA"} +{"id":2,"name":"Jane Smith","street":"48 Oak Avenue","city":"Fairview","zip":"90210","country":"USA"} +{"id":3,"name":"Michael Johnson","street":"77 Pine Road","city":"Riverside","zip":"33101","country":"USA"} +{"id":4,"name":"Emily Brown","street":"9 Cedar Street","city":"Lakeside","zip":"60601","country":"USA"} +{"id":5,"name":"Daniel Wilson","street":"642 Elm Drive","city":"Hillcrest","zip":"75201","country":"USA"} +{"id":6,"name":"Sarah Miller","street":"89 Birch Lane","city":"Greenville","zip":"85001","country":"USA"} +{"id":7,"name":"Matthew Davis","street":"501 Walnut Court","city":"Brookfield","zip":"07001","country":"USA"} +{"id":8,"name":"Olivia Garcia","street":"32 Poplar Street","city":"Kingston","zip":"30301","country":"USA"} +{"id":9,"name":"Christopher Martinez","street":"214 Aspen Drive","city":"Mount Vernon","zip":"55401","country":"USA"} +{"id":10,"name":"Sophia Rodriguez","street":"56 Sycamore Avenue","city":"Clifton","zip":"11201","country":"USA"} +{"id":11,"name":"Ethan Hernandez","street":"777 Chestnut Boulevard","city":"Oakridge","zip":"92101","country":"USA"} +{"id":12,"name":"Ava Lee","street":"14 Dogwood Circle","city":"Mapleton","zip":"63101","country":"USA"} +{"id":13,"name":"Isabella Walker","street":"205 Willow Way","city":"Brentwood","zip":"32801","country":"USA"} +{"id":14,"name":"James Hall","street":"88 Hickory Road","city":"Monroe","zip":"44101","country":"USA"} +{"id":15,"name":"Mia Allen","street":"473 Redwood Drive","city":"Pleasantville","zip":"14604","country":"USA"} +{"id":16,"name":"Alexander Young","street":"39 Palm Street","city":"Somerset","zip":"45202","country":"USA"} +{"id":17,"name":"Charlotte King","street":"120 Magnolia Lane","city":"Arlington","zip":"73101","country":"USA"} +{"id":18,"name":"Benjamin Wright","street":"68 Alder Road","city":"Ashford","zip":"53703","country":"USA"} +{"id":19,"name":"Amelia Scott","street":"311 Cypress Avenue","city":"Hamilton","zip":"27401","country":"USA"} +{"id":20,"name":"Jacob Green","street":"52 Fir Street","city":"Glenwood","zip":"97201","country":"USA"} +{"id":21,"name":"Harper Adams","street":"95 Laurel Drive","city":"Newport","zip":"84101","country":"USA"} +{"id":22,"name":"William Baker","street":"412 Spruce Court","city":"Madison","zip":"53711","country":"USA"} +{"id":23,"name":"Evelyn Nelson","street":"207 Hemlock Street","city":"Georgetown","zip":"29601","country":"USA"} +{"id":24,"name":"Logan Carter","street":"16 Juniper Road","city":"Fairmont","zip":"68502","country":"USA"} +{"id":25,"name":"Abigail Mitchell","street":"778 Maplewood Lane","city":"Salem","zip":"97301","country":"USA"} +{"id":26,"name":"Henry Perez","street":"541 Cottonwood Way","city":"Rockford","zip":"61101","country":"USA"} +{"id":27,"name":"Ella Roberts","street":"103 Beech Street","city":"Dayton","zip":"45402","country":"USA"} +{"id":28,"name":"Lucas Turner","street":"602 Aspen Court","city":"Richmond","zip":"23219","country":"USA"} +{"id":29,"name":"Luna Phillips","street":"27 Ivy Road","city":"Lexington","zip":"40502","country":"USA"} +{"id":30,"name":"Sebastian Campbell","street":"330 Grove Street","city":"Albany","zip":"12207","country":"USA"} +{"id":31,"name":"Avery Parker","street":"44 Forest Drive","city":"Trenton","zip":"08608","country":"USA"} +{"id":32,"name":"Jack Evans","street":"159 Meadow Lane","city":"Concord","zip":"03301","country":"USA"} +{"id":33,"name":"Sofia Edwards","street":"820 Orchard Street","city":"Hartford","zip":"06103","country":"USA"} +{"id":34,"name":"Levi Collins","street":"73 River Road","city":"Charleston","zip":"29401","country":"USA"} +{"id":35,"name":"Chloe Stewart","street":"18 Hilltop Drive","city":"Burlington","zip":"05401","country":"USA"} +{"id":36,"name":"Noah Sanchez","street":"926 Valley Street","city":"Aurora","zip":"80010","country":"USA"} +{"id":37,"name":"Grace Morris","street":"57 Harbor Road","city":"Dover","zip":"03820","country":"USA"} +{"id":38,"name":"Daniel Butler","street":"610 Brookside Avenue","city":"Harrisburg","zip":"17101","country":"USA"} +{"id":39,"name":"Zoey Rivera","street":"22 Highland Road","city":"Raleigh","zip":"27601","country":"USA"} +{"id":40,"name":"Wyatt Foster","street":"345 Canyon Drive","city":"Boise","zip":"83702","country":"USA"} +{"id":41,"name":"Aria Morgan","street":"91 Woodside Street","city":"Helena","zip":"59601","country":"USA"} +{"id":42,"name":"David Peterson","street":"284 Ridge Road","city":"Topeka","zip":"66603","country":"USA"} +{"id":43,"name":"Lily Bailey","street":"77 Cliff Street","city":"Cheyenne","zip":"82001","country":"USA"} +{"id":44,"name":"Joseph Ramirez","street":"402 Pinecrest Avenue","city":"Honolulu","zip":"96813","country":"USA"} +{"id":45,"name":"Scarlett Cooper","street":"55 Evergreen Road","city":"Anchorage","zip":"99501","country":"USA"} +{"id":46,"name":"Owen Reed","street":"703 Summit Avenue","city":"Billings","zip":"59101","country":"USA"} +{"id":47,"name":"Victoria Cook","street":"29 Bayshore Drive","city":"Mobile","zip":"36602","country":"USA"} +{"id":48,"name":"Samuel Rogers","street":"144 Prairie Street","city":"Tulsa","zip":"74103","country":"USA"} +{"id":49,"name":"Madison Morgan","street":"501 Sunset Drive","city":"Shreveport","zip":"71101","country":"USA"} +{"id":50,"name":"Leo Bell","street":"310 Crestview Lane","city":"Wichita","zip":"67202","country":"USA"} +{"id":51,"name":"Hannah Murphy","street":"63 Woodland Court","city":"Fargo","zip":"58102","country":"USA"} +{"id":52,"name":"Julian Murphy","street":"12 Meadowbrook Road","city":"Sioux Falls","zip":"57104","country":"USA"} +{"id":53,"name":"Aubrey Rivera","street":"245 Stonegate Drive","city":"Des Moines","zip":"50309","country":"USA"} +{"id":54,"name":"Gabriel Hughes","street":"880 Prairie View Road","city":"Omaha","zip":"68102","country":"USA"} +{"id":55,"name":"Nora Russell","street":"34 Woodland Way","city":"Lincoln","zip":"68508","country":"USA"} +{"id":56,"name":"Carter Griffin","street":"917 Maple Court","city":"Bismarck","zip":"58501","country":"USA"} +{"id":57,"name":"Riley Diaz","street":"71 Creekside Lane","city":"Pierre","zip":"57501","country":"USA"} +{"id":58,"name":"Isaac Hayes","street":"450 Silver Street","city":"Salinas","zip":"93901","country":"USA"} +{"id":59,"name":"Layla Myers","street":"23 Sunrise Drive","city":"Santa Fe","zip":"87501","country":"USA"} +{"id":60,"name":"Hudson Price","street":"566 Orchard Lane","city":"Phoenix","zip":"85004","country":"USA"} +{"id":61,"name":"Penelope Barnes","street":"99 Willowbrook Drive","city":"Tallahassee","zip":"32301","country":"USA"} +{"id":62,"name":"Anthony Ross","street":"812 Park Avenue","city":"Columbia","zip":"29201","country":"USA"} +{"id":63,"name":"Ellie Henderson","street":"47 Glenview Drive","city":"Richmond","zip":"23220","country":"USA"} +{"id":64,"name":"Dylan Coleman","street":"380 Crestwood Drive","city":"Roanoke","zip":"24011","country":"USA"} +{"id":65,"name":"Addison Jenkins","street":"121 Orchard Hill Road","city":"Knoxville","zip":"37902","country":"USA"} +{"id":66,"name":"Luke Perry","street":"604 Timber Lane","city":"Asheville","zip":"28801","country":"USA"} +{"id":67,"name":"Hazel Powell","street":"75 Ridgecrest Drive","city":"Savannah","zip":"31401","country":"USA"} +{"id":68,"name":"Andrew Long","street":"262 Cedar Crest Lane","city":"Charlottesville","zip":"22902","country":"USA"} +{"id":69,"name":"Violet Patterson","street":"81 Pine Valley Road","city":"Boulder","zip":"80302","country":"USA"} +{"id":70,"name":"Grayson Hughes","street":"130 Riverbend Drive","city":"Eugene","zip":"97401","country":"USA"} +{"id":71,"name":"Camila Flores","street":"55 Lakeview Drive","city":"Reno","zip":"89501","country":"USA"} +{"id":72,"name":"Joshua Simmons","street":"402 Forest Hill Road","city":"Spokane","zip":"99201","country":"USA"} +{"id":73,"name":"Natalie Bryant","street":"18 Creekwood Drive","city":"Tacoma","zip":"98402","country":"USA"} +{"id":74,"name":"Eli Alexander","street":"570 Briarwood Lane","city":"Bozeman","zip":"59715","country":"USA"} +{"id":75,"name":"Zoe West","street":"244 Sunset Boulevard","city":"Santa Barbara","zip":"93101","country":"USA"} +{"id":76,"name":"Lincoln Jordan","street":"67 Maple Hollow Road","city":"Pasadena","zip":"91101","country":"USA"} +{"id":77,"name":"Hannah Stone","street":"301 Highland Avenue","city":"Irvine","zip":"92614","country":"USA"} +{"id":78,"name":"Christopher Warren","street":"92 Willow Ridge Road","city":"Fresno","zip":"93721","country":"USA"} +{"id":79,"name":"Stella Boyd","street":"447 Summer Street","city":"Modesto","zip":"95354","country":"USA"} +{"id":80,"name":"Evan Reynolds","street":"36 Cedar Hill Road","city":"Stockton","zip":"95202","country":"USA"} +{"id":81,"name":"Lillian Carr","street":"710 Meadow View Lane","city":"Chandler","zip":"85225","country":"USA"} +{"id":82,"name":"Jonathan Black","street":"51 Brookhaven Drive","city":"Henderson","zip":"89011","country":"USA"} +{"id":83,"name":"Paisley Dean","street":"88 Orchard Grove Road","city":"Salt Lake City","zip":"84111","country":"USA"} +{"id":84,"name":"Adam Crawford","street":"293 Evergreen Street","city":"Provo","zip":"84601","country":"USA"} +{"id":85,"name":"Bella George","street":"137 Birchwood Lane","city":"Kansas City","zip":"64106","country":"USA"} +{"id":86,"name":"Nathan Wallace","street":"799 Sycamore Ridge Road","city":"Oklahoma City","zip":"73102","country":"USA"} +{"id":87,"name":"Skylar Cole","street":"24 Woodridge Circle","city":"New Orleans","zip":"70112","country":"USA"} +{"id":88,"name":"Miles Weber","street":"560 Bayview Road","city":"Baton Rouge","zip":"70801","country":"USA"} +{"id":89,"name":"Lucy Mills","street":"413 Valley Crest Drive","city":"Birmingham","zip":"35203","country":"USA"} +{"id":90,"name":"Aaron Nichols","street":"27 Briar Lane","city":"Louisville","zip":"40202","country":"USA"} +{"id":91,"name":"Savannah Hart","street":"962 Pine Brook Road","city":"Memphis","zip":"38103","country":"USA"} +{"id":92,"name":"Hunter Bishop","street":"590 Meadow Brook Lane","city":"Nashville","zip":"37201","country":"USA"} +{"id":93,"name":"Brooklyn Steele","street":"74 Riverwood Drive","city":"Milwaukee","zip":"53202","country":"USA"} +{"id":94,"name":"Christian Barrett","street":"158 Woodland Terrace","city":"Cincinnati","zip":"45202","country":"USA"} +{"id":95,"name":"Claire Graves","street":"803 Oak Ridge Lane","city":"Pittsburgh","zip":"15222","country":"USA"} +{"id":96,"name":"Isaiah Brewer","street":"66 Maple Grove Court","city":"Cleveland","zip":"44114","country":"USA"} +{"id":97,"name":"Audrey Walton","street":"402 Willow Creek Lane","city":"Detroit","zip":"48226","country":"USA"} +{"id":98,"name":"Elianna Page","street":"125 Brookside Court","city":"St. Louis","zip":"63103","country":"USA"} +{"id":99,"name":"Jordan Chandler","street":"380 Cedar View Road","city":"Minneapolis","zip":"55415","country":"USA"} +{"id":100,"name":"Aiden Blake","street":"97 Forest Meadow Drive","city":"Denver","zip":"80202","country":"USA"} diff --git a/test-jobs/scheduled-batch-test/connectors/tables.sqrl b/test-jobs/scheduled-batch-test/connectors/tables.sqrl new file mode 100644 index 00000000..d8ea7139 --- /dev/null +++ b/test-jobs/scheduled-batch-test/connectors/tables.sqrl @@ -0,0 +1 @@ +CREATE TABLE Addresses LIKE `addresses.jsonl`; diff --git a/test-jobs/scheduled-batch-test/scheduled-batch-package.json b/test-jobs/scheduled-batch-test/scheduled-batch-package.json new file mode 100644 index 00000000..ff8d87a2 --- /dev/null +++ b/test-jobs/scheduled-batch-test/scheduled-batch-package.json @@ -0,0 +1,24 @@ +{ + "version": "1", + "enabled-engines": ["flink", "kafka", "postgres", "vertx"], + "script": { + "main": "scheduled-batch.sqrl" + }, + "engines": { + "flink": { + "config": { + "execution.runtime-mode": "BATCH", + "security.delegation.tokens.enabled": false + }, + "deployment": { + "jobmanager-size": "small", + "taskmanager-size": "small", + "taskmanager-count": 1, + "schedule": { + "cron": "*/15 * * * *", + "timezone": "UTC" + } + } + } + } +} diff --git a/test-jobs/scheduled-batch-test/scheduled-batch.sqrl b/test-jobs/scheduled-batch-test/scheduled-batch.sqrl new file mode 100644 index 00000000..3018f4b8 --- /dev/null +++ b/test-jobs/scheduled-batch-test/scheduled-batch.sqrl @@ -0,0 +1,9 @@ +IMPORT connectors.tables.*; + +EXPORT Addresses TO print.addresses; + +AddressByCity := SELECT city, COUNT(*) AS total FROM Addresses GROUP BY city; +AddressByCountry := SELECT country, COUNT(*) AS total FROM Addresses GROUP BY country; + +EXPORT AddressByCity TO print.AddressByCity; +EXPORT AddressByCountry TO print.AddressByCountry; diff --git a/test-jobs/scheduled-batch-test/simple-scheduled-batch-package.json b/test-jobs/scheduled-batch-test/simple-scheduled-batch-package.json new file mode 100644 index 00000000..0a0a2b1c --- /dev/null +++ b/test-jobs/scheduled-batch-test/simple-scheduled-batch-package.json @@ -0,0 +1,24 @@ +{ + "version": "1", + "enabled-engines": ["flink", "kafka", "postgres", "vertx"], + "script": { + "main": "simple-scheduled-batch.sqrl" + }, + "engines": { + "flink": { + "config": { + "execution.runtime-mode": "BATCH", + "security.delegation.tokens.enabled": false + }, + "deployment": { + "jobmanager-size": "small", + "taskmanager-size": "small", + "taskmanager-count": 1, + "schedule": { + "cron": "*/15 * * * *", + "timezone": "UTC" + } + } + } + } +} diff --git a/test-jobs/scheduled-batch-test/simple-scheduled-batch.sqrl b/test-jobs/scheduled-batch-test/simple-scheduled-batch.sqrl new file mode 100644 index 00000000..2feec6f8 --- /dev/null +++ b/test-jobs/scheduled-batch-test/simple-scheduled-batch.sqrl @@ -0,0 +1,3 @@ +IMPORT connectors.tables.*; + +EXPORT Addresses TO print.addresses;