Add Wasm support

CMakeLists.txt

@@ -26,6 +26,10 @@ set(EXTENSION_SOURCES
 build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES})
 build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES})
 
+if (EMSCRIPTEN)
+  set (DUCKDB_EXTENSION_GSHEETS_LINKED_LIBS "../../vcpkg_installed/wasm32-emscripten/lib/lib*.a")
+endif()
+
 # Link OpenSSL in both the static library as the loadable extension
 target_link_libraries(${EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto)
 target_link_libraries(${LOADABLE_EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto)
@@ -35,3 +39,4 @@ install(
   EXPORT "${DUCKDB_EXPORT_SET}"
   LIBRARY DESTINATION "${INSTALL_LIB_DIR}"
   ARCHIVE DESTINATION "${INSTALL_LIB_DIR}")
+

src/gsheets_copy.cpp

@@ -177,7 +177,7 @@ unique_ptr<GlobalFunctionData> GSheetCopyFunction::GSheetWriteInitializeGlobal(C
 	if (!options.sheet.empty()) {
 		sheet_name = options.sheet;
 	} else {
-		sheet_name = get_sheet_name_from_id(spreadsheet_id, sheet_id, token);
+		sheet_name = get_sheet_name_from_id(context, spreadsheet_id, sheet_id, token);
 	}
 
 	if (!options.range.empty()) {
@@ -191,10 +191,10 @@ unique_ptr<GlobalFunctionData> GSheetCopyFunction::GSheetWriteInitializeGlobal(C
 	// Do this here in the initialization so that it only happens once
 	// OVERWRITE_RANGE takes precedence since it defaults to false and is less destructive
 	if (options.overwrite_range) {
-		delete_sheet_data(spreadsheet_id, token, encoded_sheet_name, sheet_range);
+		delete_sheet_data(context, spreadsheet_id, token, encoded_sheet_name, sheet_range);
 	} else if (options.overwrite_sheet) {
 		std::string empty_string = ""; // An empty string will clear the entire sheet
-		delete_sheet_data(spreadsheet_id, token, encoded_sheet_name, empty_string);
+		delete_sheet_data(context, spreadsheet_id, token, encoded_sheet_name, empty_string);
 	}
 
 	// Write out the headers to the file here in the Initialize so they are only written once
@@ -217,7 +217,7 @@ unique_ptr<GlobalFunctionData> GSheetCopyFunction::GSheetWriteInitializeGlobal(C
 	// If we are appending, header defaults to false
 	if (options.header) {
 		std::string response =
-		    call_sheets_api(spreadsheet_id, token, encoded_sheet_name, sheet_range, HttpMethod::POST, request_body);
+		    call_sheets_api(context, spreadsheet_id, token, encoded_sheet_name, sheet_range, HttpMethod::POST, request_body);
 
 		// Check for errors in the response
 		json response_json = parseJson(response);
@@ -249,7 +249,7 @@ void GSheetCopyFunction::GSheetWriteSink(ExecutionContext &context, FunctionData
 	if (!options.sheet.empty()) {
 		sheet_name = options.sheet;
 	} else {
-		sheet_name = get_sheet_name_from_id(gstate.spreadsheet_id, sheet_id, gstate.token);
+		sheet_name = get_sheet_name_from_id(context.client, gstate.spreadsheet_id, sheet_id, gstate.token);
 	}
 
 	if (!options.range.empty()) {
@@ -286,7 +286,7 @@ void GSheetCopyFunction::GSheetWriteSink(ExecutionContext &context, FunctionData
 	std::string request_body = sheet_data.dump();
 
 	// Make the API call to write data to the Google Sheet
-	std::string response = call_sheets_api(gstate.spreadsheet_id, gstate.token, encoded_sheet_name, sheet_range,
+	std::string response = call_sheets_api(context.client, gstate.spreadsheet_id, gstate.token, encoded_sheet_name, sheet_range,
 	                                       HttpMethod::POST, request_body);
 
 	// Check for errors in the response

src/gsheets_extension.cpp

@@ -48,10 +48,12 @@ unique_ptr<TableRef> ReadSheetReplacement(ClientContext &context, ReplacementSca
 }
 
 static void LoadInternal(ExtensionLoader &loader) {
+#ifndef EMSCRIPTEN
 	// Initialize OpenSSL
 	SSL_library_init();
 	SSL_load_error_strings();
 	OpenSSL_add_all_algorithms();
+#endif
 
 	// Register read_gsheet table function
 	TableFunction read_gsheet_function("read_gsheet", {LogicalType::VARCHAR}, ReadSheetFunction, ReadSheetBind);
@@ -67,6 +69,8 @@ static void LoadInternal(ExtensionLoader &loader) {
 	CreateGsheetSecretFunctions::Register(loader);
 	auto &config = DBConfig::GetConfig(loader.GetDatabaseInstance());
 
+	config.AddExtensionOption("gsheets_endpoint", "Alternative endpoint for the sheets.googleapis.com API",
+	                          LogicalType::VARCHAR, Value("sheets.googleapis.com"));
 	config.replacement_scans.emplace_back(ReadSheetReplacement);
 }
 

src/gsheets_get_token.cpp

@@ -122,13 +122,13 @@
 	int private_key_length = std::strlen(secret_string.c_str());
 	BIO_write(bio, private_key_pointer, private_key_length);
 	EVP_PKEY *evp_key = PEM_read_bio_PrivateKey(bio, NULL, NULL, NULL);
 	RSA *rsa = EVP_PKEY_get1_RSA(evp_key);
 
 	if (rsa != NULL) {
 		unsigned char sigret[4096] = {};
 		unsigned int siglen;
 		if (RSA_sign(NID_sha256, digest, SHA256_DIGEST_LENGTH, sigret, &siglen, rsa)) {
 			if (RSA_verify(NID_sha256, digest, SHA256_DIGEST_LENGTH, sigret, siglen, rsa)) {
 				char signature_64[1024];
 				base64encode(signature_64, (const char *)sigret, siglen);
 
@@ -137,7 +137,7 @@
 
 				std::string body =
 				    "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer&assertion=" + std::string(jwt);
-				std::string response = perform_https_request("oauth2.googleapis.com", "/token", "", HttpMethod::POST,
+				std::string response = perform_https_request(context, "oauth2.googleapis.com", "/token", "", HttpMethod::POST,
 				                                             body, "application/x-www-form-urlencoded");
 				json response_json = parseJson(response);
 				std::string token = response_json["access_token"].get<std::string>();
@@ -151,7 +151,7 @@
 			printf("RSA_sign failed: %lu, %s\n", err, ERR_error_string(err, NULL));
 		}
 
 		RSA_free(rsa);
 	}
 
 	throw InvalidInputException(

src/gsheets_read.cpp

@@ -14,10 +14,10 @@
 using json = nlohmann::json;
 
 ReadSheetBindData::ReadSheetBindData(string spreadsheet_id, string token, bool header, string sheet_name,
-                                     string sheet_range)
+                                     string sheet_range, ClientContext &context)
     : spreadsheet_id(spreadsheet_id), token(token), finished(false), row_index(0), header(header),
       sheet_name(sheet_name), sheet_range(sheet_range) {
-	response = call_sheets_api(spreadsheet_id, token, sheet_name, sheet_range, HttpMethod::GET);
+	response = call_sheets_api(context, spreadsheet_id, token, sheet_name, sheet_range, HttpMethod::GET);
 }
 
 bool IsValidNumber(const string &value) {
@@ -29,7 +29,7 @@
 	try {
 		// Try to parse as double
 		size_t processed;
 		std::stod(value, &processed);
 		// Ensure the entire string was processed
 		return processed == value.length();
 	} catch (...) {
@@ -195,7 +195,7 @@
 			}
 
 			// Validate that sheet with name exists for better error messaging
-			sheet_id = get_sheet_id_from_name(spreadsheet_id, sheet_name, token);
+			sheet_id = get_sheet_id_from_name(context, spreadsheet_id, sheet_name, token);
 		} else if (kv.first == "range") {
 			sheet_range = kv.second.GetValue<string>();
 		}
@@ -206,15 +206,15 @@
 		sheet_id = extract_sheet_id(sheet_input);
 		if (sheet_id.empty()) {
 			// Fallback to first sheet by index
-			sheet_name = get_sheet_name_from_index(spreadsheet_id, "0", token);
+			sheet_name = get_sheet_name_from_index(context, spreadsheet_id, "0", token);
 		} else {
-			sheet_name = get_sheet_name_from_id(spreadsheet_id, sheet_id, token);
+			sheet_name = get_sheet_name_from_id(context, spreadsheet_id, sheet_id, token);
 		}
 	}
 
 	std::string encoded_sheet_name = url_encode(sheet_name);
 
-	auto bind_data = make_uniq<ReadSheetBindData>(spreadsheet_id, token, header, encoded_sheet_name, sheet_range);
+	auto bind_data = make_uniq<ReadSheetBindData>(spreadsheet_id, token, header, encoded_sheet_name, sheet_range, context);
 
 	json cleanJson = parseJson(bind_data->response);
 	SheetData sheet_data = getSheetData(cleanJson);

src/gsheets_requests.cpp

@@ -1,5 +1,8 @@
 #include "gsheets_requests.hpp"
 #include "duckdb/common/exception.hpp"
+#include "duckdb/common/http_util.hpp"
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/main/database.hpp"
 #include <chrono>
 #include <openssl/ssl.h>
 #include <openssl/err.h>
@@ -8,100 +11,60 @@
 
 namespace duckdb {
 
-std::string perform_https_request(const std::string &host, const std::string &path, const std::string &token,
-                                  HttpMethod method, const std::string &body, const std::string &content_type) {
-	// NOTE: implements an exponential backoff retry strategy as per https://developers.google.com/sheets/api/limits
-	const int MAX_RETRIES = 10;
-	int retry_count = 0;
-	int backoff_s = 1;
-	while (retry_count < MAX_RETRIES) {
-		SSL_CTX *ctx = SSL_CTX_new(TLS_client_method());
-		if (!ctx) {
-			throw duckdb::IOException("Failed to create SSL context");
-		}
-
-		BIO *bio = BIO_new_ssl_connect(ctx);
-		SSL *ssl;
-		BIO_get_ssl(bio, &ssl);
-		SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);
-
-		BIO_set_conn_hostname(bio, (host + ":443").c_str());
-
-		if (BIO_do_connect(bio) <= 0) {
-			BIO_free_all(bio);
-			SSL_CTX_free(ctx);
-			throw duckdb::IOException("Failed to connect");
-		}
-
-		std::string method_str;
-		switch (method) {
-		case HttpMethod::GET:
-			method_str = "GET";
-			break;
-		case HttpMethod::POST:
-			method_str = "POST";
-			break;
-		case HttpMethod::PUT:
-			method_str = "PUT";
-			break;
-		}
-
-		std::string request = method_str + " " + path + " HTTP/1.0\r\n";
-		request += "Host: " + host + "\r\n";
-		request += "Authorization: Bearer " + token + "\r\n";
-		request += "Connection: close\r\n";
-
-		if (!body.empty()) {
-			request += "Content-Type: " + content_type + "\r\n";
-			request += "Content-Length: " + std::to_string(body.length()) + "\r\n";
-		}
-
-		request += "\r\n";
-
-		if (!body.empty()) {
-			request += body;
-		}
-
-		if (BIO_write(bio, request.c_str(), request.length()) <= 0) {
-			BIO_free_all(bio);
-			SSL_CTX_free(ctx);
-			throw duckdb::IOException("Failed to write request");
-		}
-
-		std::string response;
-		char buffer[1024];
-		int len;
-		while ((len = BIO_read(bio, buffer, sizeof(buffer))) > 0) {
-			response.append(buffer, len);
-		}
-
-		BIO_free_all(bio);
-		SSL_CTX_free(ctx);
-
-		// Check for rate limit exceeded
-		if (response.find("429 Too Many Requests") != std::string::npos) {
-			std::this_thread::sleep_for(std::chrono::seconds(backoff_s));
-			backoff_s *= 2;
-			retry_count++;
-			continue;
-		}
-
-		// Extract body from response
-		size_t body_start = response.find("\r\n\r\n");
-		if (body_start != std::string::npos) {
-			return response.substr(body_start + 4);
-		}
-
-		return response;
-	}
-	// TODO: refactor HTTPS to catch more than just rate limit exceeded or else
-	//       this could give false positives.
-	throw duckdb::IOException("Google Sheets API rate limit exceeded");
+
+std::string perform_https_request(ClientContext &context, const std::string &host, const std::string &path,
+                                  const std::string &token, HttpMethod method, const std::string &body,
+                                  const std::string &content_type) {
+      auto &db = DatabaseInstance::GetDatabase(context);
+
+       HTTPHeaders headers(db);
+       headers.Insert("Host", StringUtil::Format("%s", "http://127.0.0.1:8080/"));
+      headers.Insert("Authorization", StringUtil::Format("Bearer %s", token));
+       // headers.Insert("Connection", StringUtil::Format("close"));
+
+       if (!body.empty()) {
+               headers.Insert("Content-Type", StringUtil::Format("%s", content_type));
+               headers.Insert("Content-Length", StringUtil::Format("%s", std::to_string(body.length())));
+       }
+       auto &http_util = HTTPUtil::Get(db);
+      unique_ptr<HTTPParams> params;
+       std::string x = string("https://") + host + path;
+       params = http_util.InitializeParameters(context, x);
+       // Unclear what's peculiar about extension install flow, but those two parameters are needed
+       // to avoid lengthy retry on 304
+       params->follow_location = false;
+       params->keep_alive = false;
+
+      switch (method) {
+       case HttpMethod::GET: {
+
+               GetRequestInfo get_request(x, headers, *params, nullptr, nullptr);
+               get_request.try_request = true;
+
+               auto response = http_util.Request(get_request);
+
+               return response->body;
+
+              break;
+       }
+       case HttpMethod::POST:
+       case HttpMethod::PUT:
+               throw IOException("POST or PUT not implemented");
+              break;
+       }
+
+       // TODO: refactor HTTPS to catch more than just rate limit exceeded or else
+       //       this could give false positives.
+       throw duckdb::IOException("Google Sheets API rate limit exceeded");
 }
 
-std::string call_sheets_api(const std::string &spreadsheet_id, const std::string &token, const std::string &sheet_name,
+std::string call_sheets_api(ClientContext &context, const std::string &spreadsheet_id, const std::string &token, const std::string &sheet_name,
                             const std::string &sheet_range, HttpMethod method, const std::string &body) {
 	std::string host = "sheets.googleapis.com";
+	Value value;
+	if (context.TryGetCurrentSetting("gsheets_endpoint", value)) {
+		host = value.ToString();
+	}
 	std::string path = "/v4/spreadsheets/" + spreadsheet_id + "/values/" + sheet_name;
 
 	if (!sheet_range.empty()) {
@@ -113,21 +76,29 @@ std::string call_sheets_api(const std::string &spreadsheet_id, const std::string
 		path += "?valueInputOption=USER_ENTERED";
 	}
 
-	return perform_https_request(host, path, token, method, body);
+	return perform_https_request(context, host, path, token, method, body);
 }
 
-std::string delete_sheet_data(const std::string &spreadsheet_id, const std::string &token,
+std::string delete_sheet_data(ClientContext &context, const std::string &spreadsheet_id, const std::string &token,
                               const std::string &sheet_name, const std::string &sheet_range) {
 	std::string host = "sheets.googleapis.com";
+	Value value;
+	if (context.TryGetCurrentSetting("gsheets_endpoint", value)) {
+		host = value.ToString();
+	}
 	std::string sheet_and_range = sheet_range.empty() ? sheet_name : sheet_name + "!" + sheet_range;
 	std::string path = "/v4/spreadsheets/" + spreadsheet_id + "/values/" + sheet_and_range + ":clear";
 
-	return perform_https_request(host, path, token, HttpMethod::POST, "{}");
+	return perform_https_request(context, host, path, token, HttpMethod::POST, "{}");
 }
 
-std::string get_spreadsheet_metadata(const std::string &spreadsheet_id, const std::string &token) {
+std::string get_spreadsheet_metadata(ClientContext &context, const std::string &spreadsheet_id, const std::string &token) {
 	std::string host = "sheets.googleapis.com";
+	Value value;
+	if (context.TryGetCurrentSetting("gsheets_endpoint", value)) {
+		host = value.ToString();
+	}
 	std::string path = "/v4/spreadsheets/" + spreadsheet_id + "?&fields=sheets.properties";
-	return perform_https_request(host, path, token, HttpMethod::GET, "");
+	return perform_https_request(context, host, path, token, HttpMethod::GET, "");
 }
 } // namespace duckdb

src/gsheets_utils.cpp

@@ -51,9 +51,9 @@ std::string extract_sheet_range(const std::string &input) {
 	return "";
 }
 
-std::string get_sheet_name_from_id(const std::string &spreadsheet_id, const std::string &sheet_id,
+std::string get_sheet_name_from_id(ClientContext &context, const std::string &spreadsheet_id, const std::string &sheet_id,
                                    const std::string &token) {
-	std::string metadata_response = get_spreadsheet_metadata(spreadsheet_id, token);
+	std::string metadata_response = get_spreadsheet_metadata(context, spreadsheet_id, token);
 	json metadata = parseJson(metadata_response);
 	for (const auto &sheet : metadata["sheets"]) {
 		if (sheet["properties"]["sheetId"].get<int>() == std::stoi(sheet_id)) {
@@ -63,9 +63,9 @@ std::string get_sheet_name_from_id(const std::string &spreadsheet_id, const std:
 	throw duckdb::InvalidInputException("Sheet with ID %s not found", sheet_id);
 }
 
-std::string get_sheet_name_from_index(const std::string &spreadsheet_id, const std::string &sheet_index,
+std::string get_sheet_name_from_index(ClientContext &context, const std::string &spreadsheet_id, const std::string &sheet_index,
                                       const std::string &token) {
-	std::string metadata_response = get_spreadsheet_metadata(spreadsheet_id, token);
+	std::string metadata_response = get_spreadsheet_metadata(context, spreadsheet_id, token);
 	json metadata = parseJson(metadata_response);
 	for (const auto &sheet : metadata["sheets"]) {
 		if (sheet["properties"]["index"].get<int>() == std::stoi(sheet_index)) {
@@ -75,9 +75,9 @@ std::string get_sheet_name_from_index(const std::string &spreadsheet_id, const s
 	throw duckdb::InvalidInputException("Sheet with index %s not found", sheet_index);
 }
 
-std::string get_sheet_id_from_name(const std::string &spreadsheet_id, const std::string &sheet_name,
+std::string get_sheet_id_from_name(ClientContext &context, const std::string &spreadsheet_id, const std::string &sheet_name,
                                    const std::string &token) {
-	std::string metadata_response = get_spreadsheet_metadata(spreadsheet_id, token);
+	std::string metadata_response = get_spreadsheet_metadata(context, spreadsheet_id, token);
 	json metadata = parseJson(metadata_response);
 	for (const auto &sheet : metadata["sheets"]) {
 		if (sheet["properties"]["title"].get<std::string>() == sheet_name) {

src/include/gsheets_read.hpp

@@ -19,7 +19,7 @@ struct ReadSheetBindData : public TableFunctionData {
 	vector<LogicalType> return_types;
 	vector<string> names;
 
-	ReadSheetBindData(string spreadsheet_id, string token, bool header, string sheet_name, string sheet_range);
+	ReadSheetBindData(string spreadsheet_id, string token, bool header, string sheet_name, string sheet_range, ClientContext &context);
 };
 
 void ReadSheetFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output);