diff --git a/unfurl/server.py b/unfurl/server.py index c92d71f4..a98899d2 100644 --- a/unfurl/server.py +++ b/unfurl/server.py @@ -1,10 +1,10 @@ # Copyright (c) 2023 Adam Souzis # SPDX-License-Identifier: MIT """ -API server for the unfurl front-end app that provides JSON representations of ensembles and TOSCA service templates +API server for the unfurl front-end app that provides JSON representations of ensembles and TOSCA service templates and a patch api for updating them. -The server manage local clones of remote git repositories and uses a in-memory or redis cache for efficient access. +The server manage local clones of remote git repositories and uses a in-memory or redis cache for efficient access. """ # Security assumptions: @@ -606,11 +606,12 @@ def _do_work( work: CacheWorkCallable, latest_commit: Optional[str], cache_dependency: Optional[CacheItemDependency] = None, + include_all=False ) -> Tuple[Optional[Any], Any, CacheDirective]: try: self._deps = {} # NB: work shouldn't modify the working directory - err, value, cacheable = work(self, latest_commit) + err, value, cacheable = work(self, latest_commit, include_all=include_all) except Exception as exc: logger.error("unexpected error doing work for cache", exc_info=True) self.directives = CacheDirective(latest_commit=latest_commit, store=False) @@ -656,10 +657,11 @@ def get_or_set( latest_commit: Optional[str], validate: Optional[Callable] = None, cache_dependency: Optional[CacheItemDependency] = None, + include_all=False, ) -> Tuple[Optional[Any], Any]: if latest_commit is None and not self.stale_pull_age: # don't use the cache - return self._do_work(work, latest_commit)[0:2] + return self._do_work(work, latest_commit, include_all=include_all)[0:2] cache_value, stale = self.get_cache(cache, latest_commit) if cache_value: # cache hit @@ -706,7 +708,7 @@ def get_or_set( # there was already work inflight and use that instead return None, value - err, value, directives = self._do_work(work, latest_commit, cache_dependency) + err, value, directives = self._do_work(work, latest_commit, cache_dependency, include_all=include_all) cancel_succeeded = self._cancel_inflight(cache) # skip caching work if cancel inflight failed -- that means invalidate_cache deleted it if cancel_succeeded and not err: @@ -851,7 +853,7 @@ def format_from_path(path): def _export_cache_work( - cache_entry: CacheEntry, latest_commit: Optional[str] + cache_entry: CacheEntry, latest_commit: Optional[str], include_all=False ) -> Tuple[Any, Any, bool]: err, val = _do_export( cache_entry.project_id, @@ -860,6 +862,7 @@ def _export_cache_work( cache_entry, latest_commit, cache_entry.args or {}, + include_all=include_all ) return err, val, True @@ -895,7 +898,7 @@ def export(): return _export(request, requested_format, deployment_path) -def _export(request: Request, requested_format: str, deployment_path: str) -> Tuple[str, int]: +def _export(request: Request, requested_format: str, deployment_path: str, include_all=False) -> Tuple[str, int]: latest_commit = request.args.get("latest_commit") project_id = get_project_id(request) file_path = _get_filepath(requested_format, deployment_path) @@ -916,6 +919,7 @@ def _export(request: Request, requested_format: str, deployment_path: str) -> Tu cache, _export_cache_work, latest_commit, + include_all=include_all ) if not err: hit = cache_entry and cache_entry.hit @@ -926,7 +930,8 @@ def _export(request: Request, requested_format: str, deployment_path: str) -> Tu project_id, branch, manifest_path, "deployment", repo, args=args ) derr, djson = dcache_entry.get_or_set( - cache, _export_cache_work, latest_commit + cache, _export_cache_work, latest_commit, + include_all=include_all ) if derr: deployments.append( @@ -958,7 +963,7 @@ def _export(request: Request, requested_format: str, deployment_path: str) -> Tu @app.route("/types") def get_types(): - return _export(request, "blueprint", "dummy-ensemble.yaml") + return _export(request, "blueprint", "dummy-ensemble.yaml", include_all=True) @app.route("/populate_cache", methods=["POST"]) @@ -1067,10 +1072,11 @@ def _localenv_from_cache( deployment_path: str, latest_commit: Optional[str], args: dict, + include_all=False ) -> Tuple[Any, Optional[LocalEnv], CacheEntry]: # we want to make cloning a repo cache work to prevent concurrent cloning def _cache_localenv_work( - cache_entry: CacheEntry, latest_commit: Optional[str] + cache_entry: CacheEntry, latest_commit: Optional[str], include_all=False ) -> Tuple[Any, Any, bool]: # don't try to pull -- cache will have already pulled if latest_commit wasn't in the repo clone_location = _fetch_working_dir(cache_entry.project_id, branch, args, False) @@ -1133,10 +1139,12 @@ def _do_export( cache_entry: CacheEntry, latest_commit: Optional[str], args: dict, + include_all=False ) -> Tuple[Optional[Any], Optional[str]]: assert cache_entry.branch err, parent_localenv, localenv_cache_entry = _localenv_from_cache( - cache, project_id, cache_entry.branch, deployment_path, latest_commit, args + cache, project_id, cache_entry.branch, deployment_path, latest_commit, args, + include_all=include_all ) if err: return err, None @@ -1159,7 +1167,7 @@ def _do_export( if cache_entry: local_env.make_resolver = ServerCacheResolver.make_factory(cache_entry) exporter = getattr(to_json, "to_" + requested_format) - json_summary = exporter(local_env) + json_summary = exporter(local_env, include_all=include_all) return None, json_summary @@ -1790,7 +1798,7 @@ def load_yaml( # and track its cache entry as a dependency on the root cache entry def _work( - cache_entry: CacheEntry, latest_commit: Optional[str] + cache_entry: CacheEntry, latest_commit: Optional[str], include_all=False ) -> Tuple[Any, Any, bool]: path = os.path.join(cache_entry.checked_repo.working_dir, file_name) doc, cacheable = self._really_load_yaml(path, True, fragment) diff --git a/unfurl/to_json.py b/unfurl/to_json.py index 280721c8..b50eafca 100644 --- a/unfurl/to_json.py +++ b/unfurl/to_json.py @@ -1351,6 +1351,7 @@ def get_blueprint_from_topology(manifest: YamlManifest, db: GraphqlDB): def _to_graphql( localEnv: LocalEnv, + include_all = False ) -> Tuple[GraphqlDB, YamlManifest, GraphqlObject, ResourceTypesByName]: # set skip_validation because we want to be able to dump incomplete service templates manifest = localEnv.get_manifest(skip_validation=True, safe_mode=True) @@ -1362,7 +1363,7 @@ def _to_graphql( types_repo = tpl.get("repositories").get("types") if types_repo: # only export types, avoid built-in repositories db["repositories"] = {"types": types_repo} - types = to_graphql_nodetypes(spec) + types = to_graphql_nodetypes(spec, include_all) db["ResourceType"] = types db["ResourceTemplate"] = {} environment_instances = {} @@ -1526,8 +1527,8 @@ def add_graphql_deployment( return deployment -def to_blueprint(localEnv: LocalEnv, existing: Optional[str] = None) -> GraphqlDB: - db, manifest, env, env_types = _to_graphql(localEnv) +def to_blueprint(localEnv: LocalEnv, existing: Optional[str] = None, include_all=False) -> GraphqlDB: + db, manifest, env, env_types = _to_graphql(localEnv, include_all) assert manifest.tosca blueprint, root_name = to_graphql_blueprint(manifest.tosca, db) deployment_blueprints = get_deployment_blueprints( @@ -1663,7 +1664,7 @@ def _set_shared_instances(instances): return env_instances -def to_environments(localEnv: LocalEnv, existing: Optional[str] = None) -> GraphqlDB: +def to_environments(localEnv: LocalEnv, existing: Optional[str] = None, include_all=False) -> GraphqlDB: """ Map the environments in the project's unfurl.yaml to a json collection of Graphql objects. Each environment is be represented as: @@ -1749,7 +1750,7 @@ class Deployments(DeploymentPaths): deployments: List[GraphqlDB] -def to_deployments(localEnv: LocalEnv, existing: Optional[str] = None) -> Deployments: +def to_deployments(localEnv: LocalEnv, existing: Optional[str] = None, include_all=False) -> Deployments: assert localEnv.project db = cast(Deployments, set_deploymentpaths(localEnv.project)) deployments: List[GraphqlDB] = []