Implements Python support in miniflare.

.changeset/fuzzy-news-clean.md

@@ -0,0 +1,5 @@
+---
+"miniflare": minor
+---
+
+Implemented basic Python support

packages/miniflare/package.json

@@ -49,7 +49,7 @@
 		"glob-to-regexp": "^0.4.1",
 		"stoppable": "^1.1.0",
 		"undici": "^5.28.2",
-		"workerd": "1.20231218.0",
+		"workerd": "1.20240129.0",
 		"ws": "^8.11.0",
 		"youch": "^3.2.2",
 		"zod": "^3.20.6"

packages/miniflare/src/index.ts

@@ -1169,7 +1169,13 @@ export class Miniflare {
 			);
 		}
 
-		return { services: servicesArray, sockets, extensions };
+		const autogates = [
+			// Enables Python support in workerd.
+			// TODO(later): remove this once this gate is removed from workerd.
+			"workerd-autogate-builtin-wasm-modules"
+		];
+
+		return { services: servicesArray, sockets, extensions, autogates };
 	}
 
 	async #assembleAndUpdateConfig() {

packages/miniflare/src/plugins/core/modules.ts

@@ -46,6 +46,8 @@ export const ModuleRuleTypeSchema = z.enum([
 	"Text",
 	"Data",
 	"CompiledWasm",
+	"PythonModule",
+	"PythonRequirement"
 ]);
 export type ModuleRuleType = z.infer<typeof ModuleRuleTypeSchema>;
 
@@ -347,6 +349,12 @@ ${dim(modulesConfig)}`;
 			case "CompiledWasm":
 				this.modules.push({ name, wasm: data });
 				break;
+			case "PythonModule":
+				this.modules.push({ name, pythonModule: data.toString("utf-8") });
+				break;
+			case "PythonRequirement":
+				this.modules.push({ name, pythonRequirement: data.toString("utf-8") });
+				break;
 			default:
 				// `type` should've been validated against `ModuleRuleTypeSchema`
 				const exhaustive: never = rule.type;
@@ -405,6 +413,10 @@ export function convertModuleDefinition(
 			return { name, data: contentsToArray(contents) };
 		case "CompiledWasm":
 			return { name, wasm: contentsToArray(contents) };
+		case "PythonModule":
+			return { name, pythonModule: contentsToString(contents) };
+		case "PythonRequirement":
+			return { name, pythonRequirement: contentsToString(contents) };
 		default:
 			// `type` should've been validated against `ModuleRuleTypeSchema`
 			const exhaustive: never = def.type;
@@ -425,6 +437,8 @@ function convertWorkerModule(mod: Worker_Module): ModuleDefinition {
 	else if ("text" in m) return { path, type: "Text" };
 	else if ("data" in m) return { path, type: "Data" };
 	else if ("wasm" in m) return { path, type: "CompiledWasm" };
+	else if ("pythonModule" in m) return { path, type: "PythonModule" };
+	else if ("pythonRequirement" in m) return { path, type: "PythonRequirement" };
 
 	// This function is only used for building error messages including
 	// generated modules, and these are the types we generate.

packages/miniflare/src/runtime/config/workerd.capnp

@@ -34,7 +34,12 @@
 # afraid to fall back to code for anything the config cannot express, as Workers are very fast
 # to execute!
 
-$import "/capnp/c++.capnp".namespace("workerd::server::config");
+# Any capnp files imported here must be:
+# 1. embedded into workerd-meta.capnp
+# 2. added to `tryImportBulitin` in workerd.c++ (grep for '"/workerd/workerd.capnp"').
+using Cxx = import "/capnp/c++.capnp";
+$Cxx.namespace("workerd::server::config");
+$Cxx.allowCancellation;
 
 struct Config {
   # Top-level configuration for a workerd instance.
@@ -74,6 +79,11 @@ struct Config {
   extensions @3 :List(Extension);
   # Extensions provide capabilities to all workers. Extensions are usually prepared separately
   # and are late-linked with the app using this config field.
+
+  autogates @4 :List(Text);
+  # A list of gates which are enabled.
+  # These are used to gate features/changes in workerd and in our internal repo. See the equivalent
+  # config definition in our internal repo for more details.
 }
 
 # ========================================================================================
@@ -252,6 +262,15 @@ struct Worker {
       # (a) allows for importing Node.js-compat built-ins without the node: specifier-prefix
       # (b) exposes the subset of common Node.js globals such as process, Buffer, etc that
       #     we implement in the workerd runtime.
+
+      pythonModule @8 :Text;
+      # A Python module. All bundles containing this value type are converted into a JS/WASM Worker
+      # Bundle prior to execution.
+
+      pythonRequirement @9 :Text;
+      # A Python package that is required by this bundle. The package must be supported by
+      # Pyodide (https://pyodide.org/en/stable/usage/packages-in-pyodide.html). All packages listed
+      # will be installed prior to the execution of the worker.
     }
   }
 
@@ -485,7 +504,7 @@ struct Worker {
     }
   }
 
-  globalOutbound @6 :ServiceDesignator = (name = "internet");
+  globalOutbound @6 :ServiceDesignator = "internet";
   # Where should the global "fetch" go to? The default is the service called "internet", which
   # should usually be configured to talk to the public internet.
 
@@ -579,6 +598,9 @@ struct Worker {
 
   # TODO(someday): Support distributing objects across a cluster. At present, objects are always
   #   local to one instance of the runtime.
+
+  moduleFallback @13 :Text;
+
 }
 
 struct ExternalServer {
@@ -877,4 +899,4 @@ struct Extension {
     esModule @2 :Text;
     # Raw source code of ES module.
   }
-}
+}