Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 21 additions & 5 deletions coderd/database/dbauthz/dbauthz.go
Original file line number Diff line number Diff line change
Expand Up @@ -313,11 +313,12 @@ var (
Identifier: rbac.RoleIdentifier{Name: "jobreaper"},
DisplayName: "Job Reaper Daemon",
Site: rbac.Permissions(map[string][]policy.Action{
rbac.ResourceSystem.Type: {policy.WildcardSymbol},
rbac.ResourceTemplate.Type: {policy.ActionRead, policy.ActionUpdate},
rbac.ResourceWorkspace.Type: {policy.ActionRead, policy.ActionUpdate},
rbac.ResourceWorkspaceDormant.Type: {policy.ActionRead, policy.ActionUpdate},
rbac.ResourceProvisionerJobs.Type: {policy.ActionRead, policy.ActionUpdate},
rbac.ResourceSystem.Type: {policy.WildcardSymbol},
rbac.ResourceTemplate.Type: {policy.ActionRead, policy.ActionUpdate},
rbac.ResourceWorkspace.Type: {policy.ActionRead, policy.ActionUpdate},
rbac.ResourceWorkspaceDormant.Type: {policy.ActionRead, policy.ActionUpdate},
rbac.ResourceProvisionerJobs.Type: {policy.ActionRead, policy.ActionUpdate},
rbac.ResourceProvisionerDaemon.Type: {policy.ActionRead},
}),
User: []rbac.Permission{},
ByOrgID: map[string]rbac.OrgPermissions{},
Expand Down Expand Up @@ -4999,6 +5000,10 @@ func (q *querier) GetProvisionerJobsToBeReaped(ctx context.Context, arg database
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceProvisionerJobs); err != nil {
return nil, err
}
// This query reads provisioner daemons, so require daemon read access.
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceProvisionerDaemon); err != nil {
return nil, err
}
return q.db.GetProvisionerJobsToBeReaped(ctx, arg)
}

Expand Down Expand Up @@ -6363,6 +6368,17 @@ func (q *querier) GetWorkspacesForWorkspaceMetrics(ctx context.Context) ([]datab
return q.db.GetWorkspacesForWorkspaceMetrics(ctx)
}

func (q *querier) HasProvisionerDaemonMakingProgress(ctx context.Context, arg database.HasProvisionerDaemonMakingProgressParams) (bool, error) {
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceProvisionerJobs); err != nil {
return false, err
}
// This query reads provisioner daemons, so require daemon read access.
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceProvisionerDaemon); err != nil {
return false, err
}
return q.db.HasProvisionerDaemonMakingProgress(ctx, arg)
}

func (q *querier) HasTemplateVersionsUsingCachedModuleFileInOrg(ctx context.Context, arg database.HasTemplateVersionsUsingCachedModuleFileInOrgParams) (bool, error) {
// This query authorizes provisioner module-file downloads. The caller
// must be able to read files in the target organization; the actual
Expand Down
7 changes: 6 additions & 1 deletion coderd/database/dbauthz/dbauthz_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5779,7 +5779,12 @@ func (s *MethodTestSuite) TestSystemFunctions() {
s.Run("GetProvisionerJobsToBeReaped", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
arg := database.GetProvisionerJobsToBeReapedParams{}
dbm.EXPECT().GetProvisionerJobsToBeReaped(gomock.Any(), arg).Return([]database.ProvisionerJob{}, nil).AnyTimes()
check.Args(arg).Asserts(rbac.ResourceProvisionerJobs, policy.ActionRead)
check.Args(arg).Asserts(rbac.ResourceProvisionerJobs, policy.ActionRead, rbac.ResourceProvisionerDaemon, policy.ActionRead)
}))
s.Run("HasProvisionerDaemonMakingProgress", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
arg := database.HasProvisionerDaemonMakingProgressParams{DaemonIds: []uuid.UUID{uuid.New()}}
dbm.EXPECT().HasProvisionerDaemonMakingProgress(gomock.Any(), arg).Return(true, nil).AnyTimes()
check.Args(arg).Asserts(rbac.ResourceProvisionerJobs, policy.ActionRead, rbac.ResourceProvisionerDaemon, policy.ActionRead).Returns(true)
}))
s.Run("InsertMissingGroups", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
arg := database.InsertMissingGroupsParams{}
Expand Down
8 changes: 8 additions & 0 deletions coderd/database/dbmetrics/querymetrics.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 15 additions & 0 deletions coderd/database/dbmock/dbmock.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions coderd/database/querier.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

132 changes: 120 additions & 12 deletions coderd/database/queries.sql.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

102 changes: 94 additions & 8 deletions coderd/database/queries/provisionerjobs.sql
Original file line number Diff line number Diff line change
Expand Up @@ -326,28 +326,114 @@ WHERE
id = $1;

-- name: GetProvisionerJobsToBeReaped :many
WITH pending_requirements AS (
-- Requirement sets of pending jobs past the ordinary timeout. Jobs that
-- share an organization, provisioner type, and tags have the same
-- eligible daemons, and jobs usually share a few template tag sets, so
-- distinct sets are far fewer than pending jobs. Checking daemon
-- eligibility per set costs sets x daemons calls to the plpgsql
-- provisioner_tagset_contains function, which Postgres cannot inline,
-- instead of jobs x daemons.
SELECT DISTINCT organization_id, provisioner, tags
FROM provisioner_jobs
WHERE job_status = 'pending'::provisioner_job_status
AND updated_at < @pending_since
),
protected_requirements AS (
SELECT pr.organization_id, pr.provisioner, pr.tags
FROM pending_requirements pr
WHERE EXISTS (
SELECT 1
FROM provisioner_daemons pd
WHERE pd.organization_id = pr.organization_id
AND pr.provisioner = ANY(pd.provisioners)
AND provisioner_tagset_contains(pd.tags::tagset, pr.tags::tagset)
AND pd.last_seen_at >= @provisioner_seen_since::timestamptz
-- The running leg lets builds longer than the window count as
-- progress. Keep the legs as separate EXISTS so each is a range
-- scan on provisioner_jobs_worker_id_organization_id_completed_at_idx.
Comment thread
dylanhuff-at-coder marked this conversation as resolved.
Resolved
Comment thread
dylanhuff-at-coder marked this conversation as resolved.
Resolved
-- Keep in sync with HasProvisionerDaemonMakingProgress.
AND (
EXISTS (
SELECT 1
FROM provisioner_jobs wj
WHERE wj.worker_id = pd.id
AND wj.organization_id = pd.organization_id
AND wj.completed_at >= @provisioner_progress_since::timestamptz
Comment thread
dylanhuff-at-coder marked this conversation as resolved.
Resolved
)
OR EXISTS (
SELECT 1
FROM provisioner_jobs wj
WHERE wj.worker_id = pd.id
AND wj.organization_id = pd.organization_id
AND wj.completed_at IS NULL
AND wj.updated_at >= @hung_since
Comment thread
dylanhuff-at-coder marked this conversation as resolved.
Resolved
)
)
)
)
SELECT
*
pj.*
FROM
provisioner_jobs
provisioner_jobs pj
WHERE
(
-- If the job has not been started before @pending_since, reap it.
updated_at < @pending_since
AND started_at IS NULL
AND completed_at IS NULL
pj.updated_at < @pending_since
AND pj.started_at IS NULL
AND pj.completed_at IS NULL
-- Unless it is pending and an eligible provisioner is online and
-- making progress.
AND NOT (
pj.job_status = 'pending'::provisioner_job_status
AND EXISTS (
SELECT 1
FROM protected_requirements pr
WHERE pr.organization_id = pj.organization_id
AND pr.provisioner = pj.provisioner
AND pr.tags = pj.tags
)
)
)
OR
(
-- If the job has been started but not completed before @hung_since, reap it.
updated_at < @hung_since
AND started_at IS NOT NULL
AND completed_at IS NULL
pj.updated_at < @hung_since
AND pj.started_at IS NOT NULL
AND pj.completed_at IS NULL
)
-- To avoid repeatedly attempting to reap the same jobs, we randomly order and limit to @max_jobs.
ORDER BY random()
LIMIT @max_jobs;

-- name: HasProvisionerDaemonMakingProgress :one
Comment thread
dylanhuff-at-coder marked this conversation as resolved.
Resolved
-- Returns true when any of the given daemons is making progress, using the
-- progress rule from GetProvisionerJobsToBeReaped (keep the two in sync). It
-- does not check heartbeat or eligibility, so callers must pass only online,
-- eligible daemons.
SELECT EXISTS (
SELECT 1
FROM provisioner_daemons pd
WHERE pd.id = ANY(@daemon_ids::uuid[])
AND (
EXISTS (
SELECT 1
FROM provisioner_jobs wj
WHERE wj.worker_id = pd.id
AND wj.organization_id = pd.organization_id
AND wj.completed_at >= @provisioner_progress_since::timestamptz
)
OR EXISTS (
SELECT 1
FROM provisioner_jobs wj
WHERE wj.worker_id = pd.id
AND wj.organization_id = pd.organization_id
AND wj.completed_at IS NULL
AND wj.updated_at >= @hung_since::timestamptz
)
)
)::bool AS making_progress;

-- name: InsertProvisionerJobTimings :many
INSERT INTO provisioner_job_timings (job_id, started_at, ended_at, stage, source, action, resource)
SELECT
Expand Down
Loading
Loading