The great Blockly upgrade (#9865)

* initial blockly work

* use newblocks in webapp

* bug fixes and array block

* if/else block

* text blocks

* gridpicker

* field color number

* field slider and math_number_minmax

* duplicate on drag

* workspace context menu

* block context menu

* tooltip

* remove block dragger patch

* Fix insertion marker compile exception

* don't duplicate function definition argument reporters in domToBlock

* debugging

* Add new variable option to variable field dropdown

* Bring back the collapsed input rendering

* Monkey patch block svg to add expand button

* update some fields

* get all the rest of the field compiling

* bug fixes for fields and toolbox generation

* fix asset fields

* fix block font

* fix expandable blocks

* toolbox flyout label styling

* clean up the renderer mixin interface

* fix prompt placeholders

* make flyout buttons bigger and other fixes

* fix note editor

* Fix project importing

* shadow block hack

* give comments ids if missing

* Cache category flyouts

* fix typo in autocomplete field

* don't upgrade all shadows

* promote shadow blocks to regular blocks when they get non shadow children

* fix undo redo

* delete dead code

* fix flyout click events

* port over imagedropdown css

* remove dead code

* fix toolbox generation for draggable parameters

* remove old blocks from pxtrunner

* fix tutorial snippets

* fix variable prompt

* fix draggable decompile

* properly hide flyout when closed

* don't recycle variable blocks

* don't close flyout after creating a variable

* fix autocomplete field not resizing

* fix ts expression color and autocomplete quotes

* fix the speed and angle slider fields

* fix flyout groups with the same name as the category

* fix collapse field being added multiple times

* fix slider initial value

* fix slider dragging in firefox

* fix width on firefox

* fix position field

* fix error outline

* replace the emboss filter with a darkened outline

* fix decompiler tests

* fix karma runner

* fix karma bugs

* fix rest of karma tests

* remove old blocks

* move newblock to pxtblocks

* eliminate last references to pxtblockly

* fix editor extensions

* fix eslint and patch pxtrunner

* lint

* fix build

* fix build

* hoist type

* forgot files

* fix uglifyify

* fix test imports

* fix validation

* fixing target build

* fix duplicate on drag behavior for new functions

* fix highlight when blocks change size

* remove type reference

* fix flyout-only blockly tutorials

* fix melody editor and add fixme notes

* remove dead code and add fancy note slides

* merge conflict

* clarify fixmes

* fix toggle click target and dropdown for external inputs

* bring back menu separators in dropdown

* dotted outline for draggable params

* get textdropdown color from parent

* fix import

* fix tests

* update blockly

* fix edit function arguments

* pin new dependencies

* bump major version to 10

Author: riknoll

gulpfile.js

@@ -38,24 +38,14 @@ const pxtlib = () => compileTsProject("pxtlib");
 const pxtcompiler = () => compileTsProject("pxtcompiler");
 const pxtpy = () => compileTsProject("pxtpy");
 const pxtsim = () => compileTsProject("pxtsim");
-const pxtblocks = () => compileTsProject("pxtblocks");
 const pxtrunner = () => compileTsProject("pxtrunner", "built", true);
 const pxteditor = () => compileTsProject("pxteditor", "built", true);
 const pxtweb = () => compileTsProject("docfiles/pxtweb", "built/web");
 const backendutils = () => compileTsProject("backendutils")
 const cli = () => compileTsProject("cli", "built", true);
 const webapp = () => compileTsProject("webapp", "built", true);
 const reactCommon = () => compileTsProject("react-common", "built/react-common", true);
-
-const pxtblockly = () => gulp.src([
-    "webapp/public/blockly/blockly_compressed.js",
-    "webapp/public/blockly/blocks_compressed.js",
-    "webapp/public/blockly/plugins.js",
-    "webapp/public/blockly/msg/js/en.js",
-    "built/pxtblocks.js"
-])
-    .pipe(concat("pxtblockly.js"))
-    .pipe(gulp.dest("built"));
+const pxtblocks = () => compileTsProject("pxtblocks", "built/pxtblocks", true);
 
 const pxtapp = () => gulp.src([
     "node_modules/lzma/src/lzma_worker-min.js",
@@ -86,7 +76,6 @@ const pxtembed = () => gulp.src([
     "built/pxtlib.js",
     "built/pxtcompiler.js",
     "built/pxtpy.js",
-    "built/pxtblockly.js",
     "built/pxtsim.js",
     "built/web/runnerembed.js"
 ])
@@ -121,8 +110,8 @@ function initWatch() {
     const tasks = [
         pxtlib,
         gulp.parallel(pxtcompiler, pxtsim, backendutils),
-        gulp.parallel(pxtpy, gulp.series(copyBlockly, pxtblocks, pxtblockly)),
-        pxteditor,
+        pxtpy,
+        gulp.parallel(pxtblocks, pxteditor),
         gulp.parallel(pxtrunner, cli, pxtcommon),
         gulp.parallel(updatestrings, browserifyEmbed),
         gulp.parallel(pxtjs, pxtdts, pxtapp, pxtworker, pxtembed),
@@ -142,7 +131,7 @@ function initWatch() {
     gulp.watch("./backendutils/**/*", gulp.series(backendutils, ...tasks.slice(2)));
 
     gulp.watch("./pxtpy/**/*", gulp.series(pxtpy, ...tasks.slice(3)));
-    gulp.watch("./pxtblocks/**/*", gulp.series(gulp.series(copyBlockly, pxtblocks, pxtblockly), ...tasks.slice(3)));
+    gulp.watch("./pxtblocks/**/*", gulp.series(pxtblocks, ...tasks.slice(4)));
 
     gulp.watch("./pxteditor/**/*", gulp.series(pxteditor, ...tasks.slice(4)));
 
@@ -162,7 +151,7 @@ function initWatchCli() {
     const tasks = [
         pxtlib,
         gulp.parallel(pxtcompiler),
-        gulp.parallel(pxtpy, gulp.series(pxtblocks, pxtblockly)),
+        pxtpy,
         cli,
         notifyBuildComplete
     ]
@@ -172,7 +161,6 @@ function initWatchCli() {
     gulp.watch("./pxtcompiler/**/*", gulp.series(pxtcompiler, ...tasks.slice(2)));
 
     gulp.watch("./pxtpy/**/*", gulp.series(pxtpy, ...tasks.slice(3)));
-    gulp.watch("./pxtblockly/**/*", gulp.series(gulp.series(copyBlockly, pxtblocks, pxtblockly), ...tasks.slice(3)));
 
     gulp.watch("./cli/**/*", gulp.series(cli, ...tasks.slice(5)));
 }
@@ -388,8 +376,6 @@ const copyWebapp = () =>
         "built/pxtlib.js",
         "built/pxtcompiler.js",
         "built/pxtpy.js",
-        "built/pxtblocks.js",
-        "built/pxtblockly.js",
         "built/pxtsim.js",
         "built/webapp/src/worker.js",
         "built/webapp/src/serviceworker.js",
@@ -401,17 +387,16 @@ const copyWebapp = () =>
 const copySemanticFonts = () => gulp.src("node_modules/semantic-ui-less/themes/default/assets/fonts/*")
     .pipe(gulp.dest("built/web/fonts"))
 
-const browserifyWebapp = () => process.env.PXT_ENV == 'production' ?
-    exec('node node_modules/browserify/bin/cmd ./built/webapp/src/app.js -g [ envify --NODE_ENV production ] -g uglifyify -o ./built/web/main.js') :
-    exec('node node_modules/browserify/bin/cmd built/webapp/src/app.js -o built/web/main.js --debug')
+const execBrowserify = (entryPoint, outfile) => process.env.PXT_ENV == 'production' ?
+    exec(`node node_modules/browserify/bin/cmd ${entryPoint} -g [ envify --NODE_ENV production ] -g [ uglifyify --ignore '**/node_modules/@blockly/**' ] -o ${outfile}`) :
+    exec(`node node_modules/browserify/bin/cmd ${entryPoint} -o ${outfile} --debug`);
 
-const browserifyAssetEditor = () => process.env.PXT_ENV == 'production' ?
-    exec('node node_modules/browserify/bin/cmd ./built/webapp/src/assetEditor.js -g [ envify --NODE_ENV production ] -g uglifyify -o ./built/web/pxtasseteditor.js') :
-    exec('node node_modules/browserify/bin/cmd built/webapp/src/assetEditor.js -o built/web/pxtasseteditor.js --debug')
+const browserifyWebapp = () => execBrowserify("./built/webapp/src/app.js", "./built/web/main.js");
+
+const browserifyAssetEditor = () => execBrowserify("./built/webapp/src/assetEditor.js", "./built/web/pxtasseteditor.js");
+
+const browserifyEmbed = () => execBrowserify("./built/pxtrunner/embed.js", "./built/web/runnerembed.js");
 
-const browserifyEmbed = () => process.env.PXT_ENV == 'production' ?
-    exec('node node_modules/browserify/bin/cmd ./built/pxtrunner/embed.js -g [ envify --NODE_ENV production ] -g uglifyify -o ./built/web/runnerembed.js') :
-    exec('node node_modules/browserify/bin/cmd built/pxtrunner/embed.js -o built/web/runnerembed.js --debug')
 
 const buildSVGIcons = () => {
     let webfontsGenerator = require('@vusion/webfonts-generator')
@@ -534,37 +519,6 @@ const copyMonaco = gulp.series(
     stripMonacoSourceMaps
 );
 
-
-
-/********************************************************
-                      Blockly
-*********************************************************/
-
-const copyBlocklyCompressed = () => gulp.src([
-    "node_modules/pxt-blockly/blocks_compressed.js",
-    "node_modules/pxt-blockly/blockly_compressed.js"
-])
-    .pipe(gulp.dest("webapp/public/blockly/"));
-
-const copyBlocklyExtensions = () => gulp.src("node_modules/@blockly/**/dist/index.js")
-    .pipe(concat("plugins.js"))
-    .pipe(gulp.dest("webapp/public/blockly/"));
-
-const copyBlocklyEnJs = () => gulp.src("node_modules/pxt-blockly/msg/js/en.js")
-    .pipe(gulp.dest("webapp/public/blockly/msg/js/"));
-
-const copyBlocklyEnJson = () => gulp.src("node_modules/pxt-blockly/msg/json/en.json")
-    .pipe(gulp.dest("webapp/public/blockly/msg/json/"));
-
-const copyBlocklyMedia = () => gulp.src("node_modules/pxt-blockly/media/*")
-    .pipe(gulp.dest("webapp/public/blockly/media"))
-
-const copyBlocklyTypings = () => gulp.src("node_modules/pxt-blockly/typings/blockly.d.ts")
-    .pipe(gulp.dest("localtypings/"))
-
-const copyBlockly = gulp.parallel(copyBlocklyCompressed, copyBlocklyExtensions, copyBlocklyEnJs, copyBlocklyEnJson, copyBlocklyMedia, copyBlocklyTypings);
-
-
 /********************************************************
                       Skillmap
 *********************************************************/
@@ -764,7 +718,10 @@ const testtutorials = testTask("tutorial-test", "tutorialrunner.js");
 const testlanguageservice = testTask("language-service", "languageservicerunner.js");
 const testpxteditor = pxtEditorTestTask();
 
-const buildKarmaRunner = () => compileTsProject("tests/blocklycompiler-test", "built/tests/", true);
+const buildKarmaRunner = () => compileTsProject("tests/blocklycompiler-test", "built/", true);
+const browserifyKarma = () =>
+    exec('node node_modules/browserify/bin/cmd built/tests/blocklycompiler-test/test.spec.js -o built/tests/karma-test-runner.js --debug');
+
 const runKarma = () => {
     let command;
     if (isWin32) {
@@ -777,9 +734,11 @@ const runKarma = () => {
     }
     return exec(command, true);
 }
-const karma = gulp.series(buildKarmaRunner, runKarma);
+const karma = gulp.series(buildKarmaRunner, browserifyKarma, runKarma);
 
-const buildBlocksTestRunner = () => compileTsProject("tests/blocks-test", "built/tests", true)
+const buildBlocksTestRunner = () => compileTsProject("tests/blocks-test", "built/", true);
+const browserifyBlocksTestRunner = () =>
+    exec('node node_modules/browserify/bin/cmd built/tests/blocks-test/blocksrunner.js -o built/tests/blocksrunner.js --debug');
 
 const testAll = gulp.series(
     testdecompiler,
@@ -849,11 +808,10 @@ const buildAll = gulp.series(
     updatestrings,
     maybeUpdateWebappStrings(),
     copyTypescriptServices,
-    copyBlocklyTypings,
     gulp.parallel(pxtlib, pxtweb),
     gulp.parallel(pxtcompiler, pxtsim, backendutils),
-    gulp.parallel(pxtpy, gulp.series(copyBlockly, pxtblocks, pxtblockly)),
-    pxteditor,
+    pxtpy,
+    gulp.parallel(pxteditor, pxtblocks),
     gulp.parallel(pxtrunner, cli, pxtcommon),
     browserifyEmbed,
     gulp.parallel(pxtjs, pxtdts, pxtapp, pxtworker, pxtembed),
@@ -866,6 +824,7 @@ const buildAll = gulp.series(
     browserifyAssetEditor,
     gulp.parallel(semanticjs, copyJquery, copyWebapp, copySemanticFonts, copyMonaco),
     buildBlocksTestRunner,
+    browserifyBlocksTestRunner,
     runUglify
 );
 
@@ -876,15 +835,14 @@ exports.clean = clean;
 exports.build = buildAll;
 
 exports.webapp = gulp.series(
-    reactCommon,
+    gulp.parallel(reactCommon, pxtblocks, pxteditor),
     webapp,
     browserifyWebapp,
     browserifyAssetEditor
 )
 
 exports.skillmapTest = testSkillmap;
 exports.updatestrings = updatestrings;
-exports.updateblockly = copyBlockly;
 exports.lint = lint
 exports.testdecompiler = testdecompiler;
 exports.testlang = testlang;
@@ -915,7 +873,7 @@ exports.testpxteditor = testpxteditor;
 exports.cli = gulp.series(
     gulp.parallel(pxtlib, pxtweb),
     gulp.parallel(pxtcompiler, pxtsim, backendutils),
-    gulp.parallel(pxtpy, gulp.series(copyBlockly, pxtblocks, pxtblockly)),
+    pxtpy,
     pxteditor,
     gulp.parallel(pxtrunner, cli, pxtcommon),
     pxtjs

karma.conf.js

@@ -21,9 +21,8 @@ module.exports = function(config) {
       'built/web/typescript.js',
       'webapp/public/blockly/**/*.js',
       'built/pxtlib.js',
-      'built/pxtblocks.js',
       'built/pxtcompiler.js',
-      'built/tests/tests.spec.js',
+      'built/tests/karma-test-runner.js',
 
       // test assets
       { pattern: 'tests/blocklycompiler-test/cases/*.blocks', watched: false, included: false, served: true, nocache: false },

localtypings/evaluation.d.ts

@@ -0,0 +1,5 @@
+namespace pxt.blocks {
+    export interface EvaluationResult {
+        result: boolean;
+    }
+}

localtypings/navigationController.d.ts

@@ -0,0 +1,19 @@
+type WorkspaceSvg = import("blockly").WorkspaceSvg;
+
+declare module '@blockly/keyboard-navigation' {
+    class NavigationController {
+        init(): void;
+        addWorkspace(workspace: WorkspaceSvg): void;
+        enable(workspace: WorkspaceSvg): void;
+        disable(workspace: WorkspaceSvg): void;
+        focusToolbox(workspace: WorkspaceSvg): void;
+        navigation: Navigation;
+    }
+
+    class Navigation {
+        resetFlyout(workspace: WorkspaceSvg, shouldHide: boolean): void;
+        setState(workspace: WorkspaceSvg, state: BlocklyNavigationState): void;
+    }
+
+    type BlocklyNavigationState = "workspace" | "toolbox" | "flyout";
+}

localtypings/pxtarget.d.ts

@@ -1,6 +1,5 @@
 /// <reference path="pxtpackage.d.ts" />
 /// <reference path="pxtparts.d.ts" />
-/// <reference path="pxtblockly.d.ts" />
 /// <reference path="pxtelectron.d.ts" />
 
 declare namespace pxt {
@@ -356,7 +355,8 @@ declare namespace pxt {
         invertedMonaco?: boolean; // if true: use the vs-dark monaco theme
         invertedGitHub?: boolean; // inverted github view
         lightToc?: boolean; // if true: do NOT use inverted style in docs toc
-        blocklyOptions?: Blockly.BlocklyOptions; // Blockly options, see Configuration: https://developers.google.com/blockly/guides/get-started/web
+        // FIXME (riknoll): Can't use Blockly types here
+        blocklyOptions?: any; // Blockly options, see Configuration: https://developers.google.com/blockly/guides/get-started/web
         hideFlyoutHeadings?: boolean; // Hide the flyout headings at the top of the flyout when on a mobile device.
         monacoColors?: pxt.Map<string>; // Monaco theme colors, see https://code.visualstudio.com/docs/getstarted/theme-color-reference
         simAnimationEnter?: string; // Simulator enter animation
@@ -398,8 +398,8 @@ declare namespace pxt {
         useUploadMessage?: boolean; // change "Download" text to "Upload"
         downloadIcon?: string; // which icon io use for download
         blockColors?: Map<string>; // block namespace colors, used for build in categories
-        blockIcons?: Map<string>;
-        blocklyColors?: Blockly.Colours; // Blockly workspace, flyout and other colors
+        blockIcons?: Map<string | number>;
+        blocklyColors?: pxt.Map<string>; // Overrides for the styles in the workspace Blockly.Theme.ComponentStyle
         socialOptions?: SocialOptions; // show social icons in share dialog, options like twitter handle and org handle
         noReloadOnUpdate?: boolean; // do not notify the user or reload the page when a new app cache is downloaded
         appPathNames?: string[]; // Authorized URL paths in embedded apps, all other paths will display a warning banner