From 2a2cfee8b0d4678dc4efd4dfd4204527d4b67d1d Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Wed, 5 Aug 2020 18:25:08 +0200 Subject: [PATCH] Including the assemble files into webpack --- file.ts | 81 +++-------- package-lock.json | 35 ++++- package.json | 3 +- tsbaseconfig.json | 1 + tsconfig.json | 1 + web/app/workers/codec/OpusCodec.ts | 215 +++++++++++++++-------------- web/native-codec/CMakeLists.txt | 2 +- web/native-codec/build.sh | 1 + webpack-web.config.ts | 5 +- webpack.config.ts | 10 ++ 10 files changed, 184 insertions(+), 170 deletions(-) diff --git a/file.ts b/file.ts index 96e2c91b..95851f6f 100644 --- a/file.ts +++ b/file.ts @@ -85,42 +85,23 @@ const APP_FILE_LIST_SHARED_SOURCE: ProjectResource[] = [ "path": "img/", "local-path": "./shared/img/" - } -]; - -const APP_FILE_LIST_SHARED_VENDORS: ProjectResource[] = []; - -const APP_FILE_LIST_CLIENT_SOURCE: ProjectResource[] = [ - { /* client css files */ - "client-only": true, - "type": "css", - "search-pattern": /.*\.css$/, - "build-target": "dev|rel", - - "path": "css/", - "local-path": "./client/css/" }, - { /* client js files */ - "client-only": true, - "type": "js", - "search-pattern": /.*\.js/, - "build-target": "dev", - - "path": "js/", - "local-path": "./client/js/" - } -]; - -const APP_FILE_LIST_WEB_SOURCE: ProjectResource[] = [ - { /* generated assembly files */ + { /* assembly files */ "web-only": true, "type": "wasm", "search-pattern": /.*\.(wasm)/, "build-target": "dev|rel", - "path": "wasm/", - "local-path": "./web/native-codec/generated/" - }, + "path": "js/", + "local-path": "./dist/" + } +]; + +const APP_FILE_LIST_SHARED_VENDORS: ProjectResource[] = []; + +const APP_FILE_LIST_CLIENT_SOURCE: ProjectResource[] = []; + +const APP_FILE_LIST_WEB_SOURCE: ProjectResource[] = [ { /* translations */ "web-only": true, /* Only required for the web client */ "type": "i18n", @@ -289,8 +270,6 @@ namespace server { search_options: SearchOptions; } - const exec: (command: string) => Promise<{ stdout: string, stderr: string }> = util.promisify(cp.exec); - let files: ProjectResource[] = []; let server: http.Server; let options: Options; @@ -333,7 +312,7 @@ namespace server { } } - async function serve_file(pathname: string, query: any, response: http.ServerResponse) { + async function serve_file(pathname: string, response: http.ServerResponse) { const file = await generator.search_http_file(files, pathname, options.search_options); if(!file) { console.log("[SERVER] Client requested unknown file %s", pathname); @@ -358,7 +337,7 @@ namespace server { fis.on("data", data => response.write(data)); } - async function handle_api_request(request: http.IncomingMessage, response: http.ServerResponse, url: url_utils.UrlWithParsedQuery) { + async function handle_api_request(response: http.ServerResponse, url: url_utils.UrlWithParsedQuery) { if(url.query["type"] === "files") { response.writeHead(200, { "info-version": 1 }); response.write("type\thash\tpath\tname\n"); @@ -369,7 +348,7 @@ namespace server { } else if(url.query["type"] === "file") { let p = path.join(url.query["path"] as string, url.query["name"] as string).replace(/\\/g, "/"); if(!p.startsWith("/")) p = "/" + p; - serve_file(p, url.query, response); + await serve_file(p, response); return; } @@ -395,12 +374,12 @@ namespace server { if(url.pathname === "/api.php") { //Client API - handle_api_request(request, response, url); + handle_api_request(response, url); return; } else if(url.pathname === "/") { url.pathname = "/index.html"; } - serve_file(url.pathname, url.query, response); + serve_file(url.pathname, response); } } @@ -526,28 +505,6 @@ namespace watcher { } } - export class TSCWatcher extends Watcher { - constructor() { - super("TSC"); - //this.verbose = true; - } - - protected start_command(): string[] { - return ["npm", "run", "tsc", "--", "-w"]; - } - } - - export class SASSWatcher extends Watcher { - constructor() { - super("SASS"); - this.verbose = false; - } - - protected start_command(): string[] { - return ["npm", "run", "sass", "--", "--watch", "shared/css:shared/css"]; - } - } - export class WebPackWatcher extends Watcher { private readonly target; @@ -576,7 +533,7 @@ async function main_serve(target: "client" | "web", mode: "rel" | "dev", port: n console.log("Server started on %d", port); console.log("To stop the server press ^K^C."); - await new Promise(resolve => {}); + await new Promise(() => {}); } async function main_develop(node: boolean, target: "client" | "web", port: number, flags: string[]) { @@ -626,7 +583,7 @@ async function main_develop(node: boolean, target: "client" | "web", port: numbe async function git_tag() { const git_rev = fs.readFileSync(path.join(__dirname, ".git", "HEAD")).toString(); - let version; + if(git_rev.indexOf("/") === -1) return git_rev.substr(0, 7); else @@ -655,7 +612,7 @@ async function main_generate(target: "client" | "web", mode: "rel" | "dev", dest const exec = util.promisify(cp.exec); linker = async (source, target) => { const command = "ln -s " + source + " " + target; - const { stdout, stderr } = await exec(command); + const { stderr } = await exec(command); if(stderr) throw "failed to create link: " + stderr; } diff --git a/package-lock.json b/package-lock.json index f65b7939..db2041dc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "client", + "name": "teaspeak-web", "version": "1.2.0", "lockfileVersion": 1, "requires": true, @@ -1373,6 +1373,7 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/@types/jsrender/-/jsrender-1.0.5.tgz", "integrity": "sha512-Fjdp5QACaBMsd5vpx9x27rggFa0nyd8zqWnuTw8Aum4+gM/NiQubb6pweE3sgfHwrjRh7BGjYydpE4WYbsB+Ow==", + "dev": true, "requires": { "jsrender": "*" } @@ -4765,6 +4766,38 @@ "integrity": "sha512-0btnI/H8f2pavGMN8w40mlSKOfTK2SVJmBfBeVIj3kNw0swwgzyRq0d5TJVOwodFmtvpPeWPN/MCcfuWF0Ezbw==", "dev": true }, + "file-loader": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/file-loader/-/file-loader-6.0.0.tgz", + "integrity": "sha512-/aMOAYEFXDdjG0wytpTL5YQLfZnnTmLNjn+AIrJ/6HVnTfDqLsVKUUwkDf4I4kgex36BvjuXEn/TX9B/1ESyqQ==", + "dev": true, + "requires": { + "loader-utils": "^2.0.0", + "schema-utils": "^2.6.5" + }, + "dependencies": { + "json5": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.1.3.tgz", + "integrity": "sha512-KXPvOm8K9IJKFM0bmdn8QXh7udDh1g/giieX0NLCaMnb4hEiVFqnop2ImTXCc5e0/oHz3LTqmHGtExn5hfMkOA==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + }, + "loader-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.0.tgz", + "integrity": "sha512-rP4F0h2RaWSvPEkD7BLDFQnvSf+nK+wr3ESUjNTyAGobqrijmW92zc+SO6d4p4B1wh7+B/Jg1mkQe5NYUEHtHQ==", + "dev": true, + "requires": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^2.1.2" + } + } + } + }, "file-uri-to-path": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", diff --git a/package.json b/package.json index 21bd18ea..460e6dc3 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,6 @@ "author": "TeaSpeak (WolverinDEV)", "license": "ISC", "devDependencies": { - "@types/jsrender": "^1.0.5", "@babel/core": "^7.10.4", "@babel/plugin-transform-runtime": "^7.10.4", "@babel/preset-env": "^7.10.4", @@ -29,6 +28,7 @@ "@types/fs-extra": "^8.0.1", "@types/html-minifier": "^3.5.3", "@types/jquery": "^3.3.34", + "@types/jsrender": "^1.0.5", "@types/loader-utils": "^1.1.3", "@types/lodash": "^4.14.149", "@types/moment": "^2.13.0", @@ -48,6 +48,7 @@ "csso-cli": "^3.0.0", "ejs": "^3.0.2", "exports-loader": "^0.7.0", + "file-loader": "^6.0.0", "fs-extra": "latest", "gulp": "^4.0.2", "html-loader": "^1.0.0", diff --git a/tsbaseconfig.json b/tsbaseconfig.json index 17fc34fd..d3528081 100644 --- a/tsbaseconfig.json +++ b/tsbaseconfig.json @@ -17,6 +17,7 @@ "webpack/EJSGenerator.ts", "webpack/WatLoader.ts", "webpack/DevelBlocks.ts", + "webpack/EmscriptenLoader.ts", "loader/IndexGenerator.ts", diff --git a/tsconfig.json b/tsconfig.json index 1919ddd8..79932b27 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,6 +12,7 @@ "paths": { "*": ["shared/declarations/*"], "tc-shared/*": ["shared/js/*"], + "tc-backend/web/assembly/*": ["web/native-codec/generated/*"], /* specific web part */ "tc-backend/web/*": ["web/app/*"], /* specific web part */ "tc-backend/*": ["shared/backend.d/*"], "tc-loader": ["loader/exports/loader.d.ts"], diff --git a/web/app/workers/codec/OpusCodec.ts b/web/app/workers/codec/OpusCodec.ts index 48c36cf6..59cc2587 100644 --- a/web/app/workers/codec/OpusCodec.ts +++ b/web/app/workers/codec/OpusCodec.ts @@ -2,105 +2,105 @@ import * as cworker from "./CodecWorker"; import {CodecType} from "tc-backend/web/codec/Codec"; import {CodecWorker} from "./CodecWorker"; -declare global { - interface Window { - __init_em_module: ((Module: any) => void)[]; - } -} -self.__init_em_module = self.__init_em_module || []; - const WASM_ERROR_MESSAGES = [ 'no native wasm support detected' ]; -let Module; -self.__init_em_module.push(m => Module = m); -const runtime_initialize_promise = new Promise((resolve, reject) => { - self.__init_em_module.push(Module => { - const cleanup = () => { - Module['onRuntimeInitialized'] = undefined; - Module['onAbort'] = undefined; - }; +interface OpusModuleType extends EmscriptenModule { + cwrap: typeof cwrap; +} - Module['onRuntimeInitialized'] = () => { - cleanup(); - resolve(); - }; - - Module['onAbort'] = error => { - cleanup(); - - let message; - if(error instanceof DOMException) - message = "DOMException (" + error.name + "): " + error.code + " => " + error.message; - else if(error instanceof Error) { - message = error.message; - } else { - message = error; - } - - reject(message); - } - }); -}); - -self.__init_em_module.push(Module => { - Module['print'] = function() { - const message = arguments[0] as string; - if(message.startsWith("CompileError: WebAssembly.instantiate(): ")) { - /* Compile errors also get printed to error stream so no need to log them here */ - return; - } - console.log(...arguments); +let OpusModule = {} as OpusModuleType; +const runtimeInitializedPromise = new Promise((resolve, reject) => { + const cleanup = () => { + OpusModule['onRuntimeInitialized'] = undefined; + OpusModule['onAbort'] = undefined; }; - Module['printErr'] = function() { - const message = arguments[0] as string; - if(message.startsWith("wasm streaming compile failed: ")) { - const error_message = message.substr(31); - if(error_message.startsWith("TypeError: Failed to execute 'compile' on 'WebAssembly': ")) { - console.warn("Failed to compile opus native code: %s", error_message.substr(57)); - } else { - console.warn("Failed to prepare opus native code asynchronously: %s", error_message); - } - return; - } else if(message === "falling back to ArrayBuffer instantiation") { - /* - We suppress this message, because it comes directly after "wasm streaming compile failed:". - So if we want to print multiple lines we just have to edit the lines above. - */ - return; - } else if(message.startsWith("failed to asynchronously prepare wasm:")) { - /* - Will be handled via abort - */ - return; - } else if(message.startsWith("CompileError: WebAssembly.instantiate():")) { - /* - Will be handled via abort already - */ - return; - } - - for(const suppress of WASM_ERROR_MESSAGES) - if((arguments[0] as string).indexOf(suppress) != -1) - return; - - console.error(...arguments); + OpusModule['onRuntimeInitialized'] = () => { + cleanup(); + resolve(); }; - Module['locateFile'] = file => "../../wasm/" + file; + OpusModule['onAbort'] = error => { + cleanup(); + + let message; + if(error instanceof DOMException) + message = "DOMException (" + error.name + "): " + error.code + " => " + error.message; + else if(error instanceof Error) { + message = error.message; + } else { + message = error; + } + + reject(message); + } }); +OpusModule['print'] = function() { + const message = arguments[0] as string; + if(message.startsWith("CompileError: WebAssembly.instantiate(): ")) { + /* Compile errors also get printed to error stream so no need to log them here */ + return; + } + console.log(...arguments); +}; + +OpusModule['printErr'] = function() { + const message = arguments[0] as string; + if(message.startsWith("wasm streaming compile failed: ")) { + const error_message = message.substr(31); + if(error_message.startsWith("TypeError: Failed to execute 'compile' on 'WebAssembly': ")) { + console.warn("Failed to compile opus native code: %s", error_message.substr(57)); + } else { + console.warn("Failed to prepare opus native code asynchronously: %s", error_message); + } + return; + } else if(message === "falling back to ArrayBuffer instantiation") { + /* + We suppress this message, because it comes directly after "wasm streaming compile failed:". + So if we want to print multiple lines we just have to edit the lines above. + */ + return; + } else if(message.startsWith("failed to asynchronously prepare wasm:")) { + /* + Will be handled via abort + */ + return; + } else if(message.startsWith("CompileError: WebAssembly.instantiate():")) { + /* + Will be handled via abort already + */ + return; + } + + for(const suppress of WASM_ERROR_MESSAGES) + if((arguments[0] as string).indexOf(suppress) != -1) + return; + + console.error(...arguments); +}; + self.addEventListener("unhandledrejection", event => { + let message; if(event.reason instanceof Error) { - if(event.reason.name === "RuntimeError" && event.reason.message.startsWith("abort(CompileError: WebAssembly.instantiate():")) { - /* - We already handled that error via the Module['printErr'] callback. - */ - event.preventDefault(); + if(event.reason.name !== "RuntimeError") return; - } + else + message = event.reason.message; + } else if(typeof event.reason === "string") { + message = event.reason; + } else { + return; + } + + if(message.startsWith("abort(CompileError: WebAssembly.instantiate():")) { + /* + We already handled that error via the Module['printErr'] callback. + */ + event.preventDefault(); + return; } }); @@ -145,15 +145,15 @@ class OpusWorker implements CodecWorker { } initialise?() : string { - this.fn_newHandle = Module.cwrap("codec_opus_createNativeHandle", "number", ["number", "number"]); - this.fn_decode = Module.cwrap("codec_opus_decode", "number", ["number", "number", "number", "number"]); - this.fn_encode = Module.cwrap("codec_opus_encode", "number", ["number", "number", "number", "number"]); - this.fn_reset = Module.cwrap("codec_opus_reset", "number", ["number"]); + this.fn_newHandle = OpusModule.cwrap("codec_opus_createNativeHandle", "number", ["number", "number"]); + this.fn_decode = OpusModule.cwrap("codec_opus_decode", "number", ["number", "number", "number", "number"]); + this.fn_encode = OpusModule.cwrap("codec_opus_encode", "number", ["number", "number", "number", "number"]); + this.fn_reset = OpusModule.cwrap("codec_opus_reset", "number", ["number"]); this.nativeHandle = this.fn_newHandle(this.channelCount, this.type); - this.nativeBufferPtr = Module._malloc(OpusWorker.kProcessBufferSize); - this.processBuffer = new Uint8Array(Module.HEAPU8.buffer, this.nativeBufferPtr, OpusWorker.kProcessBufferSize); + this.nativeBufferPtr = OpusModule._malloc(OpusWorker.kProcessBufferSize); + this.processBuffer = new Uint8Array(OpusModule.HEAPU8.buffer, this.nativeBufferPtr, OpusWorker.kProcessBufferSize); return undefined; } @@ -197,19 +197,26 @@ cworker.register_codec(CodecType.OPUS_VOICE, async () => new OpusWorker(1, OpusT cworker.set_initialize_callback(async () => { try { - require("tc-generated/codec/opus"); - } catch (e) { - if(Module) { - if(typeof(Module['onAbort']) === "function") { - Module['onAbort']("Failed to load native scripts"); - } /* else the error had been already handled because its a WASM error */ - } else { - throw e; - } - } - if(!Module) - throw "Missing module handle"; + /* could be directly required since it's just a file reference */ + const [ moduleCreator, wasmFile ] = await Promise.all([ + import("tc-backend/web/assembly/TeaWeb-Worker-Codec-Opus.js"), - await runtime_initialize_promise; + // @ts-ignore + import("tc-backend/web/assembly/TeaWeb-Worker-Codec-Opus.wasm") + ]); + + const module = moduleCreator(Object.assign(OpusModule, { + locateFile(file: string) { + return file.endsWith(".wasm") ? wasmFile.default : file; + } + })); + + if(module !== OpusModule) + throw "invalid opus module object"; + } catch (e) { + OpusModule['onAbort']("Failed to load native scripts"); + } + + await runtimeInitializedPromise; return true; }); \ No newline at end of file diff --git a/web/native-codec/CMakeLists.txt b/web/native-codec/CMakeLists.txt index ead1f8c6..a2abd056 100644 --- a/web/native-codec/CMakeLists.txt +++ b/web/native-codec/CMakeLists.txt @@ -18,7 +18,7 @@ endfunction() import_opus() set(CMAKE_CXX_FLAGS "-O3 --llvm-lto 1 --memory-init-file 0 -s WASM=1 -s ASSERTIONS=1") # -s ALLOW_MEMORY_GROWTH=1 -O3 -set(CMAKE_EXE_LINKER_FLAGS "-s EXPORTED_FUNCTIONS='[\"_malloc\", \"_free\"]' -s EXTRA_EXPORTED_RUNTIME_METHODS='[\"ccall\", \"cwrap\"]' -s ENVIRONMENT='worker' --pre-js ${CMAKE_SOURCE_DIR}/init.js") # +set(CMAKE_EXE_LINKER_FLAGS "-s MODULARIZE=1 -s EXPORTED_FUNCTIONS='[\"_malloc\", \"_free\"]' -s EXTRA_EXPORTED_RUNTIME_METHODS='[\"ccall\", \"cwrap\"]' -s ENVIRONMENT='worker' --pre-js ${CMAKE_SOURCE_DIR}/init.js") # set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/generated/") add_executable(TeaWeb-Worker-Codec-Opus src/opus.cpp) diff --git a/web/native-codec/build.sh b/web/native-codec/build.sh index fcfa5219..08318e36 100644 --- a/web/native-codec/build.sh +++ b/web/native-codec/build.sh @@ -10,6 +10,7 @@ cd build_ || exit 1 emcmake cmake .. || { echo "emcmake cmake failed for the first time, trying it again" #IDKW but sometimes it does not work the first try + cd . # Sometimes helps emcmake cmake .. || { echo "Failed to execute cmake" exit 1 diff --git a/webpack-web.config.ts b/webpack-web.config.ts index 0b5d5aff..8c3e1758 100644 --- a/webpack-web.config.ts +++ b/webpack-web.config.ts @@ -8,10 +8,13 @@ export = () => config_base.config("web").then(config => { Object.assign(config.resolve.alias, { "tc-shared": path.resolve(__dirname, "shared/js"), + "tc-backend/web/assembly": path.resolve(__dirname, "web/native-codec/generated"), "tc-backend/web": path.resolve(__dirname, "web/app"), "tc-backend": path.resolve(__dirname, "web/app"), - "tc-generated/codec/opus": path.resolve(__dirname, "web/native-codec/generated/TeaWeb-Worker-Codec-Opus.js"), }); + config.node = config.node || {}; + config.node["fs"] = "empty"; + return Promise.resolve(config); }); \ No newline at end of file diff --git a/webpack.config.ts b/webpack.config.ts index 9f23ea71..f88b1086 100644 --- a/webpack.config.ts +++ b/webpack.config.ts @@ -88,6 +88,7 @@ export const config = async (target: "web" | "client"): Promise = isDevelopment: isDevelopment }) ].filter(e => !!e), + module: { rules: [ { @@ -175,6 +176,15 @@ export const config = async (target: "web" | "client"): Promise = "./webpack/WatLoader.js" ] }, + { + test: /\.wasm$/, + type: 'javascript/auto', + loader: 'file-loader', + options: { + /* the public path will already be set by emscripten base path */ + publicPath: './' + } + }, { test: /\.svg$/, loader: 'svg-inline-loader'