Wire up module loading to application startup (#21703)
* Early module loader bundler * Add a module installer script * Add dev-friendly docs * Add real module-api dependency * Speed up `yarn add` for mulitple modules * Fix version check for modules * Appease the linterpull/13/merge
Travis Ralston
3 years ago
committed by
GitHub
parent
f03200f8e6
commit
f1e5b95554
@ -1,3 +1,5 @@ |
||||
src/vector/modernizr.js |
||||
# Legacy skinning file that some people might still have |
||||
src/component-index.js |
||||
# Auto-generated file |
||||
src/modules.ts |
||||
|
||||
@ -0,0 +1,25 @@ |
||||
# A sample build_config.yaml to supply to Element Web's build pipeline, |
||||
# enabling custom functionality at compile time. Copy this file to |
||||
# `build_config.yaml` in the same directory to use, as you would with |
||||
# `config.json`. |
||||
# |
||||
# Note: The vast majority of users DO NOT need this. If you are looking |
||||
# to build your own Element Web as seen on app.element.io or similar then |
||||
# this is not required. |
||||
# |
||||
# This config file does become required if you are looking to add runtime |
||||
# functionality to Element Web, such as customisation endpoints and modules. |
||||
# |
||||
# Over time we might expand this config to better support some use cases. |
||||
# Watch the release notes for features which might impact this config. |
||||
|
||||
# The modules to install. See ./docs/modules.md for more information on |
||||
# what modules are. |
||||
# |
||||
# The values of this are provided to `yarn add` for inclusion. |
||||
modules: |
||||
# An example of pulling a module from NPM |
||||
- "@vector-im/element-web-ilag-module@^0.0.1" |
||||
|
||||
# An example of pulling a module from github |
||||
- "github:vector-im/element-web-ilag-module#main" |
||||
@ -0,0 +1,48 @@ |
||||
# Module system |
||||
|
||||
The module system in Element Web is a way to add or modify functionality of Element Web itself, bundled at compile time |
||||
for the app. This means that modules are loaded as part of the `yarn build` process but have an effect on user experience |
||||
at runtime. |
||||
|
||||
## Installing modules |
||||
|
||||
If you already have a module you want to install, such as our [ILAG Module](https://github.com/vector-im/element-web-ilag-module), |
||||
then copy `build_config.sample.yaml` to `build_config.yaml` in the same directory. In your new `build_config.yaml` simply |
||||
add the reference to the module as described by the sample file, using the same syntax you would for `yarn add`: |
||||
|
||||
```yaml |
||||
modules: |
||||
# Our module happens to be published on NPM, so we use that syntax to reference it. |
||||
- "@vector-im/element-web-ilag-module@latest" |
||||
``` |
||||
|
||||
Then build the app as you normally would: `yarn build` or `yarn dist` (if compatible on your platform). If you are building |
||||
the Docker image then ensure your `build_config.yaml` ends up in the build directory. Usually this works fine if you use |
||||
the current directory as the build context (the `.` in `docker build -t my-element-web .`). |
||||
|
||||
## Writing modules |
||||
|
||||
While writing modules is meant to be easy, not everything is possible yet. For modules which want to do something we haven't |
||||
exposed in the module API, the module API will need to be updated. This means a PR to both the |
||||
[`matrix-react-sdk`](https://github.com/matrix-org/matrix-react-sdk) and [`matrix-react-sdk-module-api`](https://github.com/matrix-org/matrix-react-sdk-module-api). |
||||
|
||||
Once your change to the module API is accepted, the `@matrix-org/react-sdk-module-api` dependency gets updated at the |
||||
`matrix-react-sdk` and `element-web` layers (usually by us, the maintainers) to ensure your module can operate. |
||||
|
||||
If you're not adding anything to the module API, or your change was accepted per above, then start off with a clone of |
||||
our [ILAG module](https://github.com/vector-im/element-web-ilag-module) which will give you a general idea for what the |
||||
structure of a module is and how it works. |
||||
|
||||
The following requirements are key for any module: |
||||
1. The module must depend on `@matrix-org/react-sdk-module-api` (usually as a dev dependency). |
||||
2. The module's `main` entrypoint must have a `default` export for the `RuntimeModule` instance, supporting a constructor |
||||
which takes a single parameter: a `ModuleApi` instance. This instance is passed to `super()`. |
||||
3. The module must be deployed in a way where `yarn add` can access it, as that is how the build system will try to |
||||
install it. Note that while this is often NPM, it can also be a GitHub/GitLab repo or private NPM registry. |
||||
|
||||
... and that's pretty much it. As with any code, please be responsible and call things in line with the documentation. |
||||
Both `RuntimeModule` and `ModuleApi` have extensive documentation to describe what is proper usage and how to set things |
||||
up. |
||||
|
||||
If you have any questions then please visit [#element-dev:matrix.org](https://matrix.to/#/#element-dev:matrix.org) on |
||||
Matrix and we'll help as best we can. |
||||
@ -0,0 +1,33 @@ |
||||
/* |
||||
Copyright 2022 New Vector Ltd. |
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); |
||||
you may not use this file except in compliance with the License. |
||||
You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
*/ |
||||
|
||||
import * as YAML from "yaml"; |
||||
import * as fs from "fs"; |
||||
|
||||
export type BuildConfig = { |
||||
// Dev note: make everything here optional for user safety. Invalid
|
||||
// configs are very possible.
|
||||
|
||||
// The module references to include in the build.
|
||||
modules?: string[]; |
||||
}; |
||||
|
||||
export function readBuildConfig(): BuildConfig { |
||||
if (fs.existsSync("./build_config.yaml")) { |
||||
return YAML.parse(fs.readFileSync("./build_config.yaml", "utf-8")); |
||||
} |
||||
return {}; // no config
|
||||
} |
||||
@ -0,0 +1,191 @@ |
||||
/* |
||||
Copyright 2022 New Vector Ltd. |
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); |
||||
you may not use this file except in compliance with the License. |
||||
You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
*/ |
||||
|
||||
import * as fs from "fs"; |
||||
import * as childProcess from "child_process"; |
||||
import * as semver from "semver"; |
||||
|
||||
import { BuildConfig } from "./BuildConfig"; |
||||
|
||||
// This expects to be run from ./scripts/install.ts
|
||||
|
||||
const moduleApiDepName = "@matrix-org/react-sdk-module-api"; |
||||
|
||||
const MODULES_TS_HEADER = ` |
||||
/* |
||||
* THIS FILE IS AUTO-GENERATED |
||||
* You can edit it you like, but your changes will be overwritten, |
||||
* so you'd just be trying to swim upstream like a salmon. |
||||
* You are not a salmon. |
||||
*/ |
||||
|
||||
import { RuntimeModule } from "@matrix-org/react-sdk-module-api/lib/RuntimeModule"; |
||||
`;
|
||||
const MODULES_TS_DEFINITIONS = ` |
||||
export const INSTALLED_MODULES: RuntimeModule[] = []; |
||||
`;
|
||||
|
||||
export function installer(config: BuildConfig): void { |
||||
if (!config.modules?.length) { |
||||
// nothing to do
|
||||
writeModulesTs(MODULES_TS_HEADER + MODULES_TS_DEFINITIONS); |
||||
return; |
||||
} |
||||
|
||||
let exitCode = 0; |
||||
|
||||
// We cheat a bit and store the current package.json and lockfile so we can safely
|
||||
// run `yarn add` without creating extra committed files for people. We restore
|
||||
// these files by simply overwriting them when we're done.
|
||||
const packageDeps = readCurrentPackageDetails(); |
||||
|
||||
// Record which optional dependencies there are currently, if any, so we can exclude
|
||||
// them from our "must be a module" assumption later on.
|
||||
const currentOptDeps = getOptionalDepNames(packageDeps.packageJson); |
||||
|
||||
try { |
||||
// Install the modules with yarn
|
||||
const yarnAddRef = config.modules.join(" "); |
||||
callYarnAdd(yarnAddRef); // install them all at once
|
||||
|
||||
// Grab the optional dependencies again and exclude what was there already. Everything
|
||||
// else must be a module, we assume.
|
||||
const pkgJsonStr = fs.readFileSync("./package.json", "utf-8"); |
||||
const optionalDepNames = getOptionalDepNames(pkgJsonStr); |
||||
const installedModules = optionalDepNames.filter(d => !currentOptDeps.includes(d)); |
||||
|
||||
// Ensure all the modules are compatible. We check them all and report at the end to
|
||||
// try and save the user some time debugging this sort of failure.
|
||||
const ourApiVersion = getTopLevelDependencyVersion(moduleApiDepName); |
||||
const incompatibleNames: string[] = []; |
||||
for (const moduleName of installedModules) { |
||||
const modApiVersion = getModuleApiVersionFor(moduleName); |
||||
if (!isModuleVersionCompatible(ourApiVersion, modApiVersion)) { |
||||
incompatibleNames.push(moduleName); |
||||
} |
||||
} |
||||
if (incompatibleNames.length > 0) { |
||||
console.error( |
||||
"The following modules are not compatible with this version of element-web. Please update the module " + |
||||
"references and try again.", |
||||
JSON.stringify(incompatibleNames, null, 4), // stringify to get prettier/complete output
|
||||
); |
||||
exitCode = 1; |
||||
return; // hit the finally{} block before exiting
|
||||
} |
||||
|
||||
// If we reach here, everything seems fine. Write modules.ts and log some output
|
||||
// Note: we compile modules.ts in two parts for developer friendliness if they
|
||||
// happen to look at it.
|
||||
console.log("The following modules have been installed: ", installedModules); |
||||
let modulesTsHeader = MODULES_TS_HEADER; |
||||
let modulesTsDefs = MODULES_TS_DEFINITIONS; |
||||
let index = 0; |
||||
for (const moduleName of installedModules) { |
||||
const importName = `Module${++index}`; |
||||
modulesTsHeader += `import ${importName} from "${moduleName}";\n`; |
||||
modulesTsDefs += `INSTALLED_MODULES.push(${importName});\n`; |
||||
} |
||||
writeModulesTs(modulesTsHeader + modulesTsDefs); |
||||
console.log("Done installing modules"); |
||||
} finally { |
||||
// Always restore package details (or at least try to)
|
||||
writePackageDetails(packageDeps); |
||||
|
||||
if (exitCode > 0) { |
||||
process.exit(exitCode); |
||||
} |
||||
} |
||||
} |
||||
|
||||
type RawDependencies = { |
||||
lockfile: string; |
||||
packageJson: string; |
||||
}; |
||||
|
||||
function readCurrentPackageDetails(): RawDependencies { |
||||
return { |
||||
lockfile: fs.readFileSync("./yarn.lock", "utf-8"), |
||||
packageJson: fs.readFileSync("./package.json", "utf-8"), |
||||
}; |
||||
} |
||||
|
||||
function writePackageDetails(deps: RawDependencies) { |
||||
fs.writeFileSync("./yarn.lock", deps.lockfile, "utf-8"); |
||||
fs.writeFileSync("./package.json", deps.packageJson, "utf-8"); |
||||
} |
||||
|
||||
function callYarnAdd(dep: string) { |
||||
// Add the module to the optional dependencies section just in case something
|
||||
// goes wrong in restoring the original package details.
|
||||
childProcess.execSync(`yarn add -O ${dep}`, { |
||||
env: process.env, |
||||
stdio: ['inherit', 'inherit', 'inherit'], |
||||
}); |
||||
} |
||||
|
||||
function getOptionalDepNames(pkgJsonStr: string): string[] { |
||||
return Object.keys(JSON.parse(pkgJsonStr)?.['optionalDependencies'] ?? {}); |
||||
} |
||||
|
||||
function findDepVersionInPackageJson(dep: string, pkgJsonStr: string): string { |
||||
const pkgJson = JSON.parse(pkgJsonStr); |
||||
const packages = { |
||||
...(pkgJson['optionalDependencies'] ?? {}), |
||||
...(pkgJson['devDependencies'] ?? {}), |
||||
...(pkgJson['dependencies'] ?? {}), |
||||
}; |
||||
return packages[dep]; |
||||
} |
||||
|
||||
function getTopLevelDependencyVersion(dep: string): string { |
||||
const dependencyTree = JSON.parse(childProcess.execSync(`npm list ${dep} --depth=0 --json`, { |
||||
env: process.env, |
||||
stdio: ['inherit', 'pipe', 'pipe'], |
||||
}).toString('utf-8')); |
||||
|
||||
/* |
||||
What a dependency tree looks like: |
||||
{ |
||||
"version": "1.10.13", |
||||
"name": "element-web", |
||||
"dependencies": { |
||||
"@matrix-org/react-sdk-module-api": { |
||||
"version": "0.0.1", |
||||
"resolved": "file:../../../matrix-react-sdk-module-api" |
||||
} |
||||
} |
||||
} |
||||
*/ |
||||
|
||||
return dependencyTree["dependencies"][dep]["version"]; |
||||
} |
||||
|
||||
function getModuleApiVersionFor(moduleName: string): string { |
||||
// We'll just pretend that this isn't highly problematic...
|
||||
// Yarn is fairly stable in putting modules in a flat hierarchy, at least.
|
||||
const pkgJsonStr = fs.readFileSync(`./node_modules/${moduleName}/package.json`, "utf-8"); |
||||
return findDepVersionInPackageJson(moduleApiDepName, pkgJsonStr); |
||||
} |
||||
|
||||
function isModuleVersionCompatible(ourApiVersion: string, moduleApiVersion: string): boolean { |
||||
if (!moduleApiVersion) return false; |
||||
return semver.satisfies(ourApiVersion, moduleApiVersion); |
||||
} |
||||
|
||||
function writeModulesTs(content: string) { |
||||
fs.writeFileSync("./src/modules.ts", content, "utf-8"); |
||||
} |
||||
@ -0,0 +1,21 @@ |
||||
/* |
||||
Copyright 2022 New Vector Ltd. |
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); |
||||
you may not use this file except in compliance with the License. |
||||
You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
*/ |
||||
|
||||
import { readBuildConfig } from "../BuildConfig"; |
||||
import { installer } from "../installer"; |
||||
|
||||
const buildConf = readBuildConfig(); |
||||
installer(buildConf); |
||||
@ -0,0 +1,14 @@ |
||||
{ |
||||
"extends": "./tsconfig.json", |
||||
"compilerOptions": { |
||||
"jsx": "preserve", |
||||
"declaration": false, |
||||
"outDir": "./lib/module_system", |
||||
"lib": [ |
||||
"es2019" |
||||
] |
||||
}, |
||||
"include": [ |
||||
"./module_system/**/*.ts" |
||||
] |
||||
} |
||||
Loading…
Reference in new issue