From 7ccdcf4c681b6681603104b08039b824cf7d48b3 Mon Sep 17 00:00:00 2001 From: Sven Efftinge Date: Mon, 7 Nov 2022 15:55:14 +0000 Subject: [PATCH] Incremental workspaces --- components/dashboard/src/projects/Project.tsx | 12 +- .../src/projects/ProjectSettings.tsx | 12 ++ .../dashboard/src/start/CreateWorkspace.tsx | 19 +-- .../gitpod-protocol/src/gitpod-service.ts | 6 +- components/gitpod-protocol/src/protocol.ts | 13 --- .../src/teams-projects-protocol.ts | 2 + .../ee/src/prebuilds/prebuild-manager.ts | 1 + .../ee/src/workspace/gitpod-server-impl.ts | 109 +++++++++--------- .../ee/src/workspace/workspace-factory.ts | 19 ++- .../src/workspace/gitpod-server-impl.ts | 33 ++++-- .../server/src/workspace/workspace-factory.ts | 10 +- 11 files changed, 128 insertions(+), 108 deletions(-) diff --git a/components/dashboard/src/projects/Project.tsx b/components/dashboard/src/projects/Project.tsx index a96b521feeb214..a39d64aa916c5b 100644 --- a/components/dashboard/src/projects/Project.tsx +++ b/components/dashboard/src/projects/Project.tsx @@ -378,17 +378,7 @@ export default function () { )} - + + ); @@ -271,10 +272,10 @@ export default class CreateWorkspace extends React.Component - this.createWorkspace(CreateWorkspaceMode.UseLastSuccessfulPrebuild) + this.createWorkspace({ allowUsingPreviousPrebuilds: true, ignoreRunningPrebuild: true }) } - onIgnorePrebuild={() => this.createWorkspace(CreateWorkspaceMode.ForceNew)} - onPrebuildSucceeded={() => this.createWorkspace(CreateWorkspaceMode.UsePrebuild)} + onIgnorePrebuild={() => this.createWorkspace({ ignoreRunningPrebuild: true })} + onPrebuildSucceeded={() => this.createWorkspace()} /> ); } diff --git a/components/gitpod-protocol/src/gitpod-service.ts b/components/gitpod-protocol/src/gitpod-service.ts index 2a73c77b0e547a..84667f96438806 100644 --- a/components/gitpod-protocol/src/gitpod-service.ts +++ b/components/gitpod-protocol/src/gitpod-service.ts @@ -12,7 +12,6 @@ import { WhitelistedRepository, WorkspaceImageBuild, AuthProviderInfo, - CreateWorkspaceMode, Token, UserEnvVarValue, Terms, @@ -422,7 +421,10 @@ export namespace GitpodServer { } export interface CreateWorkspaceOptions { contextUrl: string; - mode?: CreateWorkspaceMode; + // whether running workspaces on the same context should be ignored. If false (default) users will be asked. + ignoreRunningWorkspaceOnSameCommit?: boolean; + ignoreRunningPrebuild?: boolean; + allowUsingPreviousPrebuilds?: boolean; forceDefaultConfig?: boolean; } export interface StartWorkspaceOptions { diff --git a/components/gitpod-protocol/src/protocol.ts b/components/gitpod-protocol/src/protocol.ts index 8d2c53fbb8b537..6425966e5a3d21 100644 --- a/components/gitpod-protocol/src/protocol.ts +++ b/components/gitpod-protocol/src/protocol.ts @@ -1381,19 +1381,6 @@ export interface WorkspaceCreationResult { } export type RunningWorkspacePrebuildStarting = "queued" | "starting" | "running"; -export enum CreateWorkspaceMode { - // Default returns a running prebuild if there is any, otherwise creates a new workspace (using a prebuild if one is available) - Default = "default", - // ForceNew creates a new workspace irrespective of any running prebuilds. This mode is guaranteed to actually create a workspace - but may degrade user experience as currently runnig prebuilds are ignored. - ForceNew = "force-new", - // UsePrebuild polls the database waiting for a currently running prebuild to become available. This mode exists to handle the db-sync delay. - UsePrebuild = "use-prebuild", - // SelectIfRunning returns a list of currently running workspaces for the context URL if there are any, otherwise falls back to Default mode - SelectIfRunning = "select-if-running", - // UseLastSuccessfulPrebuild returns ... - UseLastSuccessfulPrebuild = "use-last-successful-prebuild", -} - export namespace WorkspaceCreationResult { export function is(data: any): data is WorkspaceCreationResult { return ( diff --git a/components/gitpod-protocol/src/teams-projects-protocol.ts b/components/gitpod-protocol/src/teams-projects-protocol.ts index eeb772bbad9e37..8b482f87124623 100644 --- a/components/gitpod-protocol/src/teams-projects-protocol.ts +++ b/components/gitpod-protocol/src/teams-projects-protocol.ts @@ -17,6 +17,8 @@ export interface ProjectSettings { useIncrementalPrebuilds?: boolean; usePersistentVolumeClaim?: boolean; keepOutdatedPrebuildsRunning?: boolean; + // whether new workspaces can start on older prebuilds and incrementally update + allowUsingPreviousPrebuilds?: boolean; } export interface Project { diff --git a/components/server/ee/src/prebuilds/prebuild-manager.ts b/components/server/ee/src/prebuilds/prebuild-manager.ts index 757c78dc637898..898dd947f43f98 100644 --- a/components/server/ee/src/prebuilds/prebuild-manager.ts +++ b/components/server/ee/src/prebuilds/prebuild-manager.ts @@ -191,6 +191,7 @@ export class PrebuildManager { const workspace = await this.workspaceFactory.createForContext( { span }, user, + project, prebuildContext, context.normalizedContextURL!, ); diff --git a/components/server/ee/src/workspace/gitpod-server-impl.ts b/components/server/ee/src/workspace/gitpod-server-impl.ts index d2a1c5b73f9333..5606bc74c005ec 100644 --- a/components/server/ee/src/workspace/gitpod-server-impl.ts +++ b/components/server/ee/src/workspace/gitpod-server-impl.ts @@ -28,7 +28,6 @@ import { WorkspaceTimeoutValues, SetWorkspaceTimeoutResult, WorkspaceContext, - CreateWorkspaceMode, WorkspaceCreationResult, PrebuiltWorkspaceContext, CommitContext, @@ -976,7 +975,8 @@ export class GitpodServerEEImpl extends GitpodServerImpl { parentCtx: TraceContext, user: User, context: WorkspaceContext, - mode: CreateWorkspaceMode, + ignoreRunningPrebuild?: boolean, + allowUsingPreviousPrebuilds?: boolean, ): Promise { const ctx = TraceContext.childContext("findPrebuiltWorkspace", parentCtx); try { @@ -989,29 +989,38 @@ export class GitpodServerEEImpl extends GitpodServerImpl { const logCtx: LogContext = { userId: user.id }; const cloneUrl = context.repository.cloneUrl; let prebuiltWorkspace: PrebuiltWorkspace | undefined; + const logPayload = { + allowUsingPreviousPrebuilds, + ignoreRunningPrebuild, + cloneUrl, + commit: commitSHAs, + prebuiltWorkspace, + }; if (OpenPrebuildContext.is(context)) { prebuiltWorkspace = await this.workspaceDb.trace(ctx).findPrebuildByID(context.openPrebuildID); - if (prebuiltWorkspace?.cloneURL !== cloneUrl) { + if ( + prebuiltWorkspace?.cloneURL !== cloneUrl && + (ignoreRunningPrebuild || prebuiltWorkspace?.state === "available") + ) { // prevent users from opening arbitrary prebuilds this way - they must match the clone URL so that the resource guards are correct. return; } } else { - prebuiltWorkspace = await this.workspaceDb - .trace(ctx) - .findPrebuiltWorkspaceByCommit(cloneUrl, commitSHAs); - } - - const logPayload = { mode, cloneUrl, commit: commitSHAs, prebuiltWorkspace }; - log.debug(logCtx, "Looking for prebuilt workspace: ", logPayload); - if (prebuiltWorkspace?.state !== "available" && mode === CreateWorkspaceMode.UseLastSuccessfulPrebuild) { - const { config } = await this.configProvider.fetchConfig({}, user, context); - const history = await this.incrementalPrebuildsService.getCommitHistoryForContext(context, user); - prebuiltWorkspace = await this.incrementalPrebuildsService.findGoodBaseForIncrementalBuild( - context, - config, - history, - user, - ); + log.debug(logCtx, "Looking for prebuilt workspace: ", logPayload); + if (!allowUsingPreviousPrebuilds) { + prebuiltWorkspace = await this.workspaceDb + .trace(ctx) + .findPrebuiltWorkspaceByCommit(cloneUrl, commitSHAs); + } else { + const { config } = await this.configProvider.fetchConfig({}, user, context); + const history = await this.incrementalPrebuildsService.getCommitHistoryForContext(context, user); + prebuiltWorkspace = await this.incrementalPrebuildsService.findGoodBaseForIncrementalBuild( + context, + config, + history, + user, + ); + } } if (!prebuiltWorkspace) { return; @@ -1026,13 +1035,9 @@ export class GitpodServerEEImpl extends GitpodServerImpl { }; return result; } else if (prebuiltWorkspace.state === "queued" || prebuiltWorkspace.state === "building") { - if (mode === CreateWorkspaceMode.ForceNew) { + if (ignoreRunningPrebuild) { // in force mode we ignore running prebuilds as we want to start a workspace as quickly as we can. return; - // TODO(janx): Fall back to parent prebuild instead, if it's available: - // const buildWorkspace = await this.workspaceDb.trace({span}).findById(prebuiltWorkspace.buildWorkspaceId); - // const parentPrebuild = await this.workspaceDb.trace({span}).findPrebuildByID(buildWorkspace.basedOnPrebuildId); - // Also, make sure to initialize it by both printing the parent prebuild logs AND re-runnnig the before/init/prebuild tasks. } const workspaceID = prebuiltWorkspace.buildWorkspaceId; @@ -1097,36 +1102,34 @@ export class GitpodServerEEImpl extends GitpodServerImpl { const inSameCluster = wsi.region === this.config.installationShortname; if (!inSameCluster) { - if (mode === CreateWorkspaceMode.UsePrebuild) { - /* We need to wait for this prebuild to finish before we return from here. - * This creation mode is meant to be used once we have gone through default mode, have confirmation from the - * message bus that the prebuild is done, and now only have to wait for dbsync to come through. Thus, - * in this mode we'll poll the database until the prebuild is ready (or we time out). - * - * Note: This polling mechanism only makes sense if the prebuild runs in cluster different from ours. - * Otherwise there's no dbsync inbetween that we might have to wait for. - * - * DB sync interval is 2 seconds at the moment, we wait ten "ticks" for the data to be synchronized. - */ - const finishedPrebuiltWorkspace = await this.pollDatabaseUntilPrebuildIsAvailable( - ctx, - prebuiltWorkspace.id, - 20000, + /* We need to wait for this prebuild to finish before we return from here. + * This creation mode is meant to be used once we have gone through default mode, have confirmation from the + * message bus that the prebuild is done, and now only have to wait for dbsync to come through. Thus, + * in this mode we'll poll the database until the prebuild is ready (or we time out). + * + * Note: This polling mechanism only makes sense if the prebuild runs in cluster different from ours. + * Otherwise there's no dbsync inbetween that we might have to wait for. + * + * DB sync interval is 2 seconds at the moment, we wait ten "ticks" for the data to be synchronized. + */ + const finishedPrebuiltWorkspace = await this.pollDatabaseUntilPrebuildIsAvailable( + ctx, + prebuiltWorkspace.id, + 20000, + ); + if (!finishedPrebuiltWorkspace) { + log.warn( + logCtx, + "did not find a finished prebuild in the database despite waiting long enough after msgbus confirmed that the prebuild had finished", + logPayload, ); - if (!finishedPrebuiltWorkspace) { - log.warn( - logCtx, - "did not find a finished prebuild in the database despite waiting long enough after msgbus confirmed that the prebuild had finished", - logPayload, - ); - return; - } else { - return { - title: context.title, - originalContext: context, - prebuiltWorkspace: finishedPrebuiltWorkspace, - } as PrebuiltWorkspaceContext; - } + return; + } else { + return { + title: context.title, + originalContext: context, + prebuiltWorkspace: finishedPrebuiltWorkspace, + } as PrebuiltWorkspaceContext; } } diff --git a/components/server/ee/src/workspace/workspace-factory.ts b/components/server/ee/src/workspace/workspace-factory.ts index 504572c17f4e1a..3dbc222a74d2d5 100644 --- a/components/server/ee/src/workspace/workspace-factory.ts +++ b/components/server/ee/src/workspace/workspace-factory.ts @@ -18,6 +18,7 @@ import { WithSnapshot, WithPrebuild, OpenPrebuildContext, + Project, } from "@gitpod/gitpod-protocol"; import { log } from "@gitpod/gitpod-protocol/lib/util/logging"; import { LicenseEvaluator } from "@gitpod/licensor/lib"; @@ -62,16 +63,17 @@ export class WorkspaceFactoryEE extends WorkspaceFactory { public async createForContext( ctx: TraceContext, user: User, + project: Project | undefined, context: WorkspaceContext, normalizedContextURL: string, ): Promise { if (StartPrebuildContext.is(context)) { return this.createForStartPrebuild(ctx, user, context, normalizedContextURL); } else if (PrebuiltWorkspaceContext.is(context)) { - return this.createForPrebuiltWorkspace(ctx, user, context, normalizedContextURL); + return this.createForPrebuiltWorkspace(ctx, user, project, context, normalizedContextURL); } - return super.createForContext(ctx, user, context, normalizedContextURL); + return super.createForContext(ctx, user, project, context, normalizedContextURL); } protected async createForStartPrebuild( @@ -146,6 +148,7 @@ export class WorkspaceFactoryEE extends WorkspaceFactory { ws = await this.createForPrebuiltWorkspace( { span }, user, + project, incrementalPrebuildContext, normalizedContextURL, ); @@ -166,7 +169,7 @@ export class WorkspaceFactoryEE extends WorkspaceFactory { if (!ws) { // No suitable parent prebuild was found -- create a (fresh) full prebuild. - ws = await this.createForCommit({ span }, user, commitContext, normalizedContextURL); + ws = await this.createForCommit({ span }, user, project, commitContext, normalizedContextURL); } ws.type = "prebuild"; ws.projectId = project?.id; @@ -217,6 +220,7 @@ export class WorkspaceFactoryEE extends WorkspaceFactory { protected async createForPrebuiltWorkspace( ctx: TraceContext, user: User, + project: Project | undefined, context: PrebuiltWorkspaceContext, normalizedContextURL: string, ): Promise { @@ -234,14 +238,19 @@ export class WorkspaceFactoryEE extends WorkspaceFactory { span.log({ error: `No build workspace with ID ${buildWorkspaceID} found - falling back to original context`, }); - return await this.createForContext({ span }, user, context.originalContext, normalizedContextURL); + return await this.createForContext( + { span }, + user, + project, + context.originalContext, + normalizedContextURL, + ); } const config = { ...buildWorkspace.config }; config.vscode = { extensions: (config && config.vscode && config.vscode.extensions) || [], }; - const project = await this.projectDB.findProjectByCloneUrl(context.prebuiltWorkspace.cloneURL); let projectId: string | undefined; // associate with a project, if it's the personal project of the current user if (project?.userId && project?.userId === user.id) { diff --git a/components/server/src/workspace/gitpod-server-impl.ts b/components/server/src/workspace/gitpod-server-impl.ts index 234664a05b707c..dc16808cb6c0ba 100644 --- a/components/server/src/workspace/gitpod-server-impl.ts +++ b/components/server/src/workspace/gitpod-server-impl.ts @@ -30,7 +30,6 @@ import { AuthProviderInfo, CommitContext, Configuration, - CreateWorkspaceMode, DisposableCollection, GetWorkspaceTimeoutResult, GitpodClient as GitpodApiClient, @@ -1053,12 +1052,11 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { traceAPIParams(ctx, { options }); const contextUrl = options.contextUrl; - const mode = options.mode || CreateWorkspaceMode.Default; let normalizedContextUrl: string = ""; let logContext: LogContext = {}; try { - const user = this.checkAndBlockUser("createWorkspace", { mode }); + const user = this.checkAndBlockUser("createWorkspace", { options }); await this.checkTermsAcceptance(); const envVars = this.userDB.getEnvVars(user.id); @@ -1070,7 +1068,7 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { normalizedContextUrl = this.contextParser.normalizeContextURL(contextUrl); let runningForContextPromise: Promise = Promise.resolve([]); const contextPromise = this.contextParser.handle(ctx, user, normalizedContextUrl); - if (mode === CreateWorkspaceMode.SelectIfRunning) { + if (!options.ignoreRunningWorkspaceOnSameCommit) { runningForContextPromise = this.findRunningInstancesForContext( ctx, contextPromise, @@ -1153,14 +1151,22 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { } } - if (mode === CreateWorkspaceMode.SelectIfRunning && context.forceCreateNewWorkspace !== true) { + if (!options.ignoreRunningWorkspaceOnSameCommit && !context.forceCreateNewWorkspace) { const runningForContext = await runningForContextPromise; if (runningForContext.length > 0) { return { existingWorkspaces: runningForContext }; } } - - const prebuiltWorkspace = await this.findPrebuiltWorkspace(ctx, user, context, mode); + const project = CommitContext.is(context) + ? await this.projectDB.findProjectByCloneUrl(context.repository.cloneUrl) + : undefined; + const prebuiltWorkspace = await this.findPrebuiltWorkspace( + ctx, + user, + context, + options.ignoreRunningPrebuild, + options.allowUsingPreviousPrebuilds || project?.settings?.allowUsingPreviousPrebuilds, + ); if (WorkspaceCreationResult.is(prebuiltWorkspace)) { ctx.span?.log({ prebuild: "running" }); return prebuiltWorkspace as WorkspaceCreationResult; @@ -1170,7 +1176,13 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { context = prebuiltWorkspace; } - const workspace = await this.workspaceFactory.createForContext(ctx, user, context, normalizedContextUrl); + const workspace = await this.workspaceFactory.createForContext( + ctx, + user, + project, + context, + normalizedContextUrl, + ); await this.mayStartWorkspace(ctx, user, workspace, runningInstancesPromise); try { await this.guardAccess({ kind: "workspace", subject: workspace }, "create"); @@ -1242,10 +1254,11 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable { } protected async findPrebuiltWorkspace( - ctx: TraceContext, + parentCtx: TraceContext, user: User, context: WorkspaceContext, - mode: CreateWorkspaceMode, + ignoreRunningPrebuild?: boolean, + allowUsingPreviousPrebuilds?: boolean, ): Promise { // prebuilds are an EE feature return undefined; diff --git a/components/server/src/workspace/workspace-factory.ts b/components/server/src/workspace/workspace-factory.ts index 9e412f8f3ede53..7af31a49cc5020 100644 --- a/components/server/src/workspace/workspace-factory.ts +++ b/components/server/src/workspace/workspace-factory.ts @@ -10,6 +10,7 @@ import { CommitContext, IssueContext, PrebuiltWorkspaceContext, + Project, PullRequestContext, Repository, SnapshotContext, @@ -38,13 +39,14 @@ export class WorkspaceFactory { public async createForContext( ctx: TraceContext, user: User, + project: Project | undefined, context: WorkspaceContext, normalizedContextURL: string, ): Promise { if (SnapshotContext.is(context)) { return this.createForSnapshot(ctx, user, context); } else if (CommitContext.is(context)) { - return this.createForCommit(ctx, user, context, normalizedContextURL); + return this.createForCommit(ctx, user, project, context, normalizedContextURL); } log.error({ userId: user.id }, "Couldn't create workspace for context", context); throw new Error("Couldn't create workspace for context"); @@ -106,16 +108,14 @@ export class WorkspaceFactory { protected async createForCommit( ctx: TraceContext, user: User, + project: Project | undefined, context: CommitContext, normalizedContextURL: string, ) { const span = TraceContext.startSpan("createForCommit", ctx); try { - const [{ config, literalConfig }, project] = await Promise.all([ - this.configProvider.fetchConfig({ span }, user, context), - this.projectDB.findProjectByCloneUrl(context.repository.cloneUrl), - ]); + const { config, literalConfig } = await this.configProvider.fetchConfig({ span }, user, context); const imageSource = await this.imageSourceProvider.getImageSource(ctx, user, context, config); if (config._origin === "derived" && literalConfig) { (context as any as AdditionalContentContext).additionalFiles = { ...literalConfig };