OwlCyberSecurity - MANAGER

Edit File: backup_jobs_helper

#!/usr/local/cpanel/3rdparty/bin/perl

#                                      Copyright 2025 WebPros International, LLC
#                                                            All rights reserved.
# copyright@cpanel.net                                          http://cpanel.net
# This code is subject to the cPanel license. Unauthorized copying is prohibited.

package scripts::backup_jobs_helper;

use cPstrict;

use Getopt::Long qw(GetOptions);
use Pod::Usage   qw(pod2usage);
use Time::Piece  ();
use Cpanel::Logger;
use Cpanel::Debug ();

use Cpanel::Plugins::FileLock            ();
use Whostmgr::CometBackup::BackupJobs    ();
use Whostmgr::CometBackup::DailyCoverage ();
use Whostmgr::CometBackup::Scheduling    ();
use Whostmgr::CometBackup::Constants     ();
use Data::UUID                           ();
use POSIX                                ();

# Longest interval wins when several jobs come due on the same tick. Needed
# because the SPA exposes no time field, so every job it creates defaults to
# 02:00 and "first due job" was decided by created_timestamp DESC -- newest
# created, which is arbitrary. An unrecognized frequency sorts last.
my %SCHEDULE_TIER_RANK = (
    monthly => 0,
    weekly  => 1,
    daily   => 2,
);

my $LOCK_FILE = '/var/run/comet_backup_jobs_helper.lock';

my $logger = Cpanel::Logger->new();

=encoding utf-8

=head1 NAME

backup_jobs_helper - Process scheduled backup jobs for Comet Backup integration

=head1 SYNOPSIS

backup_jobs_helper [options]

=head1 DESCRIPTION

This script processes scheduled backup jobs by checking which jobs are due to run
and executing them. It is designed to be called periodically by cron.

The script performs the following operations:

=over

=item * Retrieves all active backup jobs from the database

=item * Checks which enabled jobs are scheduled to run at the current time
(a disabled job is never a candidate)

=item * Orders the due jobs longest-interval-first (monthly, weekly, daily) and
skips any whose accounts, destinations and data types a backup that FINISHED earlier today
already copied, recording a C<skipped> run for each so the skip is visible and the
job's C<next_run> moves to the next eligible day
(L<Whostmgr::CometBackup::DailyCoverage>)

=item * Executes the first remaining due backup job

=item * Records the dispatched run's C<run_uuid> on the job; the runner itself
records the job's status and last run time

=item * On ticks where no runner is active, runs the once-per-calendar-day
grouped-file maintenance pass and consumes the one-shot C<backup-tool> restart
flag when nothing is running (L<Whostmgr::CometBackup::AgentTmpdir>)

=item * Skips its dispatch sweep entirely while that restart's dispatch
interlock is held (C<$Constants::AGENT_RESTART_LOCK>), since a run dispatched
into an agent restart is killed with the agent

=back

=head1 OPTIONS

=over

=item --help

Show this help message and exit.

=item --verbose

Enable verbose logging output.

=item --debug

Enable debug logging output (shows detailed scheduling calculations).

=item --dry-run

Show what would be done without actually executing backup jobs.

=back

=head1 EXAMPLES

# Process backup jobs normally
    /usr/local/cpanel/scripts/backup_jobs_helper

# Show what would be done without executing
    /usr/local/cpanel/scripts/backup_jobs_helper --dry-run

# Enable verbose output
    /usr/local/cpanel/scripts/backup_jobs_helper --verbose

# Enable debug output
    /usr/local/cpanel/scripts/backup_jobs_helper --debug

=cut

sub main {

# Plugin policy: nothing in this process should be world-readable.
    umask 0027;

my ( $help, $verbose, $dry_run, $debug );

GetOptions(
        'help|h'    => \$help,
        'verbose|v' => \$verbose,
        'dry-run'   => \$dry_run,
        'debug|d'   => \$debug,
    ) or pod2usage(2);

pod2usage(1) if $help;

# Enable debug logging if --debug flag is set
    if ($debug) {
        $Cpanel::Debug::level = 1;
    }

my $lock_obj = Cpanel::Plugins::FileLock->new($LOCK_FILE);

if ( -e $LOCK_FILE ) {
        if ( $lock_obj->exists() ) {

# SafeFile confirms the lock is held by a live process — exit cleanly.
            $logger->info('Another backup_jobs_helper is already running; exiting.');
            exit 0;    ## no critic (Cpanel::NoExitsFromSubroutines)
        }

# Lock file exists but SafeFile does not consider it held (previous
        # process died without cleaning up) — remove the stale file so
        # create() can succeed.
        $logger->warn("Removing stale lock (previous process is no longer running)");
        unlink $LOCK_FILE or $logger->warn("Could not unlink stale lock $LOCK_FILE: $!");
    }

my $lock = $lock_obj->create();
    unless ($lock) {
        $logger->error("Failed to acquire lock $LOCK_FILE; cannot proceed.");
        exit 1;
    }

# Disable DESTROY-based release immediately — we manage cleanup manually.
    $lock_obj->disable_auto_cleanup();

my $current_time = time();
    my $current_dt   = Time::Piece->new($current_time);

if ($verbose) {
        $logger->info( "Starting backup jobs helper at " . $current_dt->strftime('%Y-%m-%d %H:%M:%S') );
    }

eval {
        my $jobs_manager = Whostmgr::CometBackup::BackupJobs->new();
        my $scheduler    = Whostmgr::CometBackup::Scheduling->new();

# Get all active scheduled backup jobs. active_only is load-bearing:
        # without it a DISABLED job that is due becomes a candidate, the runner
        # refuses it ("Backup job is not active") before it records anything,
        # its last_run never moves, and because only the FIRST due candidate is
        # dispatched it blocks every active job ordered after it, every tick.
        my @active_jobs = $jobs_manager->get_backup_jobs( [], { active_only => 1 } );

if ($verbose) {
            $logger->info( "Found " . scalar(@active_jobs) . " active backup jobs" );
        }

# Comet has rejected this server's credentials (typically: the server
        # was recovered onto another one, which rotated them). Dispatching a run
        # would only fail it, and every request it makes counts against the
        # account's limiter, which then refuses the NEW server's valid requests.
        # Skip the sweep until the breaker's next retry; the listing above is
        # what probes Comet when that time comes, and a success resumes.
        if ( _comet_credentials_paused() ) {
            print "Backup requests are paused: this server's backup credentials are no longer valid.\n" if $dry_run;
            return;
        }

# comet_backup_runner enforces a system-wide single-runner guard via
        # its PID file: a second runner invocation will exit immediately. If a
        # runner is already active we must not mark any due job 'running' or
        # fork a second runner — the runner would die on its PID-file guard
        # and the DB row would be left stuck. Bail out of this sweep; the
        # cron tick will retry once the current runner finishes.
        if ( !$dry_run ) {
            if ( my $active_pid = Whostmgr::CometBackup::Constants::existing_runner_pid() ) {
                $logger->info("comet_backup_runner already active (PID $active_pid); skipping this sweep.");
                return;
            }
        }

# CPANEL-54636: once-per-calendar-day grouped-file maintenance (group
        # reconcile + retired-legacy-PI purge). Runs only when no runner is active
        # (checked just above) so it never races a runner's own group reconcile,
        # and is guarded by a timestamp file so it fires at most once per calendar
        # day -- no new cron surface.
        _run_daily_grouped_file_maintenance( $logger, $dry_run );

# The one-shot backup-tool restart that applies a newly installed TMPDIR.
        # Same placement class as the daily maintenance above and for the same
        # reason: it is only reachable on a tick where no runner is active (the
        # guard above returns otherwise), and ONLY the plugin knows whether Comet
        # is idle -- a systemd path/timer unit could watch the env file but would
        # have no idea whether a backup or restore is in flight, and ExecStop
        # pkills the agent's job children.
        #
        # Almost every tick returns immediately: the flag is absent unless a
        # plugin install armed it, and consuming it removes it.
        #
        # eval-wrapped like the maintenance pass: this is housekeeping and must
        # never cost a cron tick or a backup run. The module is fail-soft
        # throughout; the eval is the belt to its braces.
        eval {
            require Whostmgr::CometBackup::AgentTmpdir;
            Whostmgr::CometBackup::AgentTmpdir::maybe_restart_agent_for_tmpdir( dry_run => $dry_run );
            1;
        } or $logger->warn("backup-tool TMPDIR restart check failed: $@");

# The DISPATCH side of that same interlock, and it is an ACQUISITION
        # rather than a look: this tick takes the SHARED lease and HOLDS it
        # across the dispatch below, so the restart actor -- this process a
        # moment ago, a support script, or a future one -- cannot take its
        # EXCLUSIVE lease between the moment this sweep decides to dispatch and
        # the moment the runner it started exists. The runner inherits the
        # descriptor across exec (see _execute_backup_job), so the lease then
        # lives exactly as long as the runner does.
        #
        # Restarting backup-tool kills whatever it is executing, so dispatching a
        # run into that window would fail it for certain rather than merely race.
        #
        # SKIP, never fail: this sweep runs every five minutes and the condition
        # clears in seconds, so the next tick simply picks the job up. Logged at
        # info once, like the single-runner guard above it, and gated on
        # !$dry_run for the same reason -- a dry run performs no dispatch to
        # defer, and taking a lease would needlessly defer a real restart.
        my $restart_lease;

if ( !$dry_run ) {
            $restart_lease = Whostmgr::CometBackup::Constants::acquire_agent_restart_dispatch_lease();

if ( $restart_lease->{refused} ) {
                $logger->info('backup-tool is being restarted to apply a new TMPDIR; skipping this dispatch sweep. The next cron tick retries.');
                return;
            }
        }

my $jobs_processed    = 0;
        my $jobs_skipped      = 0;
        my $runner_dispatched = 0;

# ONE COPY PER CALENDAR DAY, PER SERVER.
        #
        # The "at most once per calendar day" rule already existed, but it was
        # scoped PER JOB: Scheduling::calculate_next_run compares THAT job's own
        # last_run against today, so one job's run could never suppress another's.
        # Field-observed consequence: a Weekly job started 02:00 and ran ~7 hours,
        # a Daily job came due ~09:00, both were legitimately due the same day, and
        # the server took two full passes over the same data.
        #
        # This is the missing server-wide half. Coverage is built ONCE per tick from
        # the runs that FINISHED today, each judged on the scope it recorded for
        # itself; a due job is skipped only when a single one of those runs covers
        # BOTH its accounts AND its destinations. Everything doubtful runs -- see
        # Whostmgr::CometBackup::DailyCoverage.
        my $coverage = _daily_coverage();

my @due_candidates;

for my $job (@active_jobs) {

$logger->debug( "LOOKING AT JOB: " . $job->{job_id} . " ( " . $job->{description} . " | $job->{schedule_type} )" );

# Skip manual jobs - they are not scheduled
            next if $job->{schedule_type} eq 'manual';

# Skip if job is currently running
            next if $job->{status} && $job->{status} eq 'running';

# Check if this job should run now
            next unless _should_job_run_now( $job, $scheduler );

push @due_candidates, $job;
        }

for my $job ( _order_due_candidates(@due_candidates) ) {

# A due job whose data another job already copied today is recorded as
            # skipped and the walk CONTINUES: the next candidate may well carry
            # accounts or destinations that copy never touched, and it is the one
            # this tick should dispatch.
            if ( my $crediting = Whostmgr::CometBackup::DailyCoverage::job_is_covered( $job, $coverage ) ) {
                _record_skipped_job( $job, $crediting, $dry_run );
                $jobs_skipped++;
                next;
            }

if ($verbose) {
                $logger->info( "Processing backup job: " . $job->{name} . " (job_id: " . $job->{job_id} . ")" );
            }

if ($dry_run) {
                print "Would execute backup job: " . $job->{name} . " (job_id: " . $job->{job_id} . ")\n";
                $jobs_processed++;
                next;
            }

# Only one runner can be active system-wide. Dispatch the
            # first due job, then stop — additional due jobs will be
            # picked up by the next cron tick once this runner exits.
            _execute_backup_job( $jobs_manager, $job, $restart_lease ? $restart_lease->{handle} : undef );
            $jobs_processed++;
            $runner_dispatched = 1;
            last;
        }

# Close THIS process's copy of the shared lease. When a runner was
        # dispatched its inherited duplicate keeps the lease alive; when none
        # was, closing the last duplicate releases it. Never an explicit unlock
        # -- that would release the runner's copy too.
        Whostmgr::CometBackup::Constants::release_agent_restart_dispatch_lease( $restart_lease->{handle} ) if $restart_lease;

if ($verbose) {
            $logger->info("Processed $jobs_processed backup jobs");
            $logger->info("Skipped $jobs_skipped due backup jobs already covered by today's backups") if $jobs_skipped;
            if ($runner_dispatched) {
                $logger->info("Deferred remaining due jobs to subsequent cron ticks (single-runner policy).");
            }
        }

if ( ( $verbose || $debug ) && $dry_run && $jobs_processed == 0 ) {
            $logger->info("No backup jobs are currently due to run");
        }
    };

$lock_obj->remove($lock);

if ($@) {
        $logger->error("Error in backup jobs helper: $@");
        exit 1;
    }

if ($verbose) {
        $logger->info("Backup jobs helper completed successfully");
    }
}

# True while Comet requests with the stored credentials are paused. Logs the
# paused notice at most once per paused window. Fails open: a breaker that
# cannot be read never stops a backup.
sub _comet_credentials_paused {
    my $paused = eval {
        require Whostmgr::CometBackup::CometBackupAPI;
        require Whostmgr::CometBackup::AuthBreaker;
        return 0 if !Whostmgr::CometBackup::CometBackupAPI::stored_credentials_paused();
        Whostmgr::CometBackup::AuthBreaker::note_skip();
        1;
    };
    return $paused ? 1 : 0;
}

sub _should_job_run_now {
    my ( $job, $scheduler ) = @_;

# Parse job configuration
    my $schedule_info = $job->{schedule_info} || {};
    my $last_run      = $job->{last_run};

# Calculate next run time based on schedule
    my $next_run_time;

eval { $next_run_time = $scheduler->calculate_next_run( $schedule_info, $last_run ); };

if ($@) {
        $logger->warn( "Could not calculate next run time for job " . $job->{job_id} . ": $@" );
        return 0;
    }

# calculate_next_run legitimately returns undef -- an unrecognized frequency, an
    # invalid time, a monthly day that exists in no upcoming month -- and the return
    # below already handles that. Rendering it must too: `localtime(undef)` warns
    # and then prints the epoch, which reads as a real answer.
    $logger->debug( "Next run time for job " . $job->{job_id} . " ( $job->{description} ) : " . ( defined $next_run_time ? scalar localtime($next_run_time) : 'none' ) );

# Re-sample the clock immediately before comparing so that any gap between
    # script start and the scheduler's own Time::Piece->new() call (which
    # returns $now for never-run jobs) cannot make the comparison permanently
    # false by a few seconds.
    return defined $next_run_time && time() >= $next_run_time;
}

# Longest interval first: monthly, then weekly, then daily, then anything whose
# frequency we do not recognize. Order WITHIN a tier is the incoming order, which
# is get_backup_jobs' created_timestamp DESC -- preserved explicitly with an index
# tiebreak rather than relying on sort() being stable.
sub _order_due_candidates {
    my (@candidates) = @_;

return @candidates if @candidates < 2;

return map { $_->[2] }
      sort { $a->[0] <=> $b->[0] || $a->[1] <=> $b->[1] }
      map { [ _schedule_tier_rank( $candidates[$_] ), $_, $candidates[$_] ] } 0 .. $#candidates;
}

sub _schedule_tier_rank {
    my ($job) = @_;

my $schedule_info = $job->{schedule_info};
    return scalar keys %SCHEDULE_TIER_RANK unless ref $schedule_info eq 'HASH';

my $frequency = $schedule_info->{frequency};
    return scalar keys %SCHEDULE_TIER_RANK unless defined $frequency && length $frequency;

return $SCHEDULE_TIER_RANK{ lc $frequency } // scalar keys %SCHEDULE_TIER_RANK;
}

# Today's coverage, built ONCE per tick: the runs that produced a real copy today,
# each carrying the scope IT RECORDED for itself.
#
# Only 'completed' and 'partial' credit the day. A failed or cancelled run copied
# nothing worth suppressing another job over, so it leaves the day open.
#
# THE DAY IS CREDITED BY THE COMPLETION DAY, NOT THE START DAY. end_time and the
# per-job guard's last_run_timestamp are the SAME EPOCH from the same time() call:
# comet_backup_runner::_complete_run_log writes $current_time to backup_runs.end_time
# and hands that same scalar to BackupJobs::update_last_run_timestamp. So keying this
# rule to end_time makes it agree with the per-job rule by construction, and a run
# that started 23:00 and completed 06:00 consumes the day it FINISHED. Keyed
# to start_time instead, that run credits nothing today and the next tick dispatches
# an overlapping daily job moments after a full copy landed: the exact duplicate work
# this exists to prevent, merely shifted across midnight. A row with no end_time is
# not a finished run and cannot credit anything.
#
# A run's coverage is read from what THAT RUN RECORDED -- user_runs rows and
# item_runs destination ids via get_recorded_run_scopes, and the content scope the
# runner stamped on its own backup_runs row at run start -- and never from its job's
# configuration as that configuration stands now. The configuration is mutable and
# the run is not: an all_accounts run that covered only alice would otherwise appear
# to have covered a bob created after it finished, and an edit to the job's accounts,
# destinations or backup_options would claim coverage the run never made. All of them
# lose an account its only snapshot of the day. A run whose scope was not recorded
# credits NOTHING.
#
# CPANEL-56041: the content half is what stops a NARROW job's run consuming the day
# for a broader one. A databases-only job and a full job routinely share accounts, a
# vault and the 02:00 tick, and without it the full job was recorded skipped without
# its last_run_timestamp moving -- so it was skipped again the next day, and every
# day after that, while its home directories, mail and settings went unbacked up.
#
# ACCEPTED TRADE-OFF, and this is where the crediting happens: the recorded scope is
# what the run ATTEMPTED, not what it achieved -- a user_runs row is counted whatever
# became of it -- so an account that failed inside a 'partial' run is not retried by
# a different job later the same day. Requiring 'completed' would be worse rather
# than stricter: one failed account in a 27-account run would trigger a whole second
# pass over the other 26. The crediting run's uuid goes into the skip reason so an
# operator can see exactly which run consumed the day and what it actually managed.
#
# Any failure at all yields empty coverage, i.e. every due job runs.
# CPANEL-56043: run a run-log READ against the lazily-constructed handle, and if it
# fails for ANY reason, ensure the schema and run it exactly once more.
#
# Withholding the constructor's ensure means the sweep no longer repairs a run log
# on its way past, and "the file exists" is not the same question as "the schema is
# current". Two shapes reach the gap: an installation upgraded from a release older
# than a run-log schema addition (a populated database missing the newer table), and
# an empty database file -- get_backup_runs, the Activity feed's reader, connects
# WITHOUT the -f guard the narrow readers have, and SQLite's connect() creates the
# file, so a server that has never completed a backup can still acquire a zero-table
# one. The eager constructor used to heal both here; without a repair, the first
# cron tick after such an upgrade would die "no such table" before log_skipped_run
# ever reached its own self-ensure.
#
# The retry is NOT conditioned on the error text. Matching DBD's "no such table"
# wording would miss a missing COLUMN and would break the moment that wording
# changes; ensuring and retrying cannot miss a schema shape. It costs nothing in the
# steady state, because it only ever runs after a read has already failed -- which
# is the whole point: the 288 healthy ticks a day still pay for no integrity_check.
# A second failure propagates to _daily_coverage's own eval, which fails open.
sub _read_run_log_with_schema_repair {
    my ( $run_logs, $read ) = @_;

my $result = eval { $read->() };

return $result unless $@;

$logger->info( "A run-log read failed; ensuring the run-log schema and retrying once: " . $@ );

# Memo-respecting, and a no-op when the database file does not exist -- so a
    # second failure in the same tick cannot pay for the sweep twice.
    $run_logs->ensure_schema();

return $read->();
}

sub _daily_coverage {

my $coverage = eval {
        require Whostmgr::CometBackup::BackupRunLogs;
        require Whostmgr::CometBackup::BackupJobs;

my $now = time();

# Resolved ONCE per tick: a destination the runner has disabled
        # server-wide is filtered out before dispatch (comet_backup_runner:3204-
        # 3235) and so can never appear in any crediting run's RECORDED scope --
        # leaving it in a candidate's list would make that job unmatchable
        # against any real run for as long as the destination stays disabled.
        # A read that fails leaves this empty, which is the fail-open direction:
        # nothing gets filtered out of any candidate's destinations, exactly the
        # (safe, over-cautious) behaviour before this existed.
        my @disabled_destination_ids = eval { Whostmgr::CometBackup::BackupJobs->new()->get_disabled_destination_ids() };

# CPANEL-56043: NO SCHEMA ENSURE ON THE FIVE-MINUTE PATH. The default
        # constructor ends in PRAGMA integrity_check, which reads the whole run-log
        # file -- several hundred MB on a server with a year of backups behind it --
        # and the once-per-process memo that makes that cheap inside the backup
        # runner buys nothing for a process cron starts and stops 288 times a day.
        # Both reads below are guarded on the file existing and never create it, so
        # there is nothing here for the ensure to make safe.
        my $run_logs = Whostmgr::CometBackup::BackupRunLogs->new( ensure_schema => 0 );

my $runs = _read_run_log_with_schema_repair(
            $run_logs,
            sub {
                return $run_logs->get_backup_runs_since(
                    since        => Whostmgr::CometBackup::DailyCoverage::calendar_day_floor($now),
                    since_column => 'end_time',
                    statuses     => [ 'completed', 'partial' ],
                );
            }
        );

my @finished_today = grep {

# The SQL floor is over end_time now too, so its width bounds only how
            # far back a FINISH can be read, not how long a run may have taken to
            # get there -- a run that started a week ago and finished in the last
            # two days is still read. This authoritative test is the localtime
            # year + day-of-year comparison the once-per-day maintenance guard
            # below already uses.
            defined $_->{end_time}
              && Whostmgr::CometBackup::DailyCoverage::is_same_calendar_day( $_->{end_time}, $now )
        } @{ $runs || [] };

my @run_ids = grep { defined $_ } map { $_->{id} } @finished_today;

# ONE batched read of both halves for every crediting run, not one per run.
        my $scopes = @run_ids ? _read_run_log_with_schema_repair( $run_logs, sub { return $run_logs->get_recorded_run_scopes( run_ids => \@run_ids ) } ) : {};

my @crediting_runs;

for my $run (@finished_today) {
            my $recorded = $scopes->{ $run->{id} // '' } // {};

push @crediting_runs, {
                job_uuid     => $run->{job_uuid},
                run_uuid     => $run->{run_uuid},
                status       => $run->{status},
                accounts     => $recorded->{accounts},
                destinations => $recorded->{destinations},

# CPANEL-56041: the third recorded half, and the only one that
                # rides the backup_runs row itself -- get_backup_runs_since hands
                # it back already decoded. A run that carries none (one taken
                # before the column existed, or whose stamp failed) credits
                # nothing at all, so every job due after it runs.
                content_scope => $run->{content_scope},
            };
        }

Whostmgr::CometBackup::DailyCoverage::compute_coverage( \@crediting_runs, disabled_destination_ids => \@disabled_destination_ids );
    };

if ( !$coverage || ref $coverage ne 'HASH' ) {
        $logger->warn( "Could not determine today's backup coverage; every due job will run: " . ( $@ || 'no coverage returned' ) );
        return Whostmgr::CometBackup::DailyCoverage::compute_coverage( [] );
    }

my $crediting_count = scalar @{ $coverage->{jobs} || [] };

$logger->info( "Today's backup coverage: $crediting_count crediting run(s), " . scalar( keys %{ $coverage->{accounts} || {} } ) . " account(s), " . scalar( keys %{ $coverage->{destinations} || {} } ) . " destination(s)" ) if $crediting_count;

return $coverage;
}

# Record that a due job was skipped because its data was already copied today.
#
# The row is written at most ONCE per (job, calendar day): cron ticks 288 times a
# day and re-decides the skip on every one of them, so without the existence check
# a single covered job would fill the Activity feed. Writing it at all is what makes
# the skip visible, and BackupJobs::get_backup_jobs reads the same rows back so
# next_run advertises the next eligible day instead of an overdue past time.
#
# Nothing here is fatal. A skip we could not record is a bookkeeping loss; failing
# the tick over it would cost the backups.
sub _record_skipped_job {
    my ( $job, $crediting, $dry_run ) = @_;

my $job_id   = $job->{job_id};
    my $job_name = $job->{name} // $job_id // 'unnamed job';

my $crediting_job    = $crediting->{job_uuid} // 'unknown';
    my $crediting_run    = $crediting->{run_uuid} // 'unknown';
    my $crediting_status = $crediting->{status}   // 'unknown';

my $reason = "Skipped: backup job $crediting_job already produced a copy of these accounts, destinations and data types today " . "(run $crediting_run, status $crediting_status). Only one backup copy per calendar day is taken.";

$logger->info( "Skipping due backup job: $job_name (job_id: " . ( $job_id // 'UNSET' ) . ") - already covered today by job $crediting_job (run_uuid: $crediting_run, status: $crediting_status)" );

if ($dry_run) {
        print "Would skip backup job: $job_name (job_id: " . ( $job_id // 'UNSET' ) . ") - already covered today by job $crediting_job (run_uuid: $crediting_run)\n";
        return;
    }

eval {
        require Whostmgr::CometBackup::BackupRunLogs;
        require Data::UUID;

# CPANEL-56043: same reasoning as _daily_coverage. The idempotence read below
        # carries the same schema repair, and log_skipped_run ensures the schema
        # ITSELF -- it has to, because the dedup decision runs on servers where no
        # backup has ever completed, so this can be the first thing ever to write to
        # the run log. (Reaching here at all means _daily_coverage's reads already
        # succeeded, so the repair is belt-and-braces rather than the load-bearing
        # copy; it is applied uniformly so no site depends on that ordering.)
        my $run_logs = Whostmgr::CometBackup::BackupRunLogs->new( ensure_schema => 0 );
        my $now      = time();

my $recorded = _read_run_log_with_schema_repair(
            $run_logs,
            sub {
                return $run_logs->get_backup_runs_since(
                    since    => Whostmgr::CometBackup::DailyCoverage::calendar_day_floor($now),
                    statuses => ['skipped'],
                    job_uuid => $job_id,
                );
            }
        );

my @today = grep { Whostmgr::CometBackup::DailyCoverage::is_same_calendar_day( $_->{start_time}, $now ) } @{ $recorded || [] };

if (@today) {
            $logger->debug( "Skip for job " . ( $job_id // 'UNSET' ) . " is already recorded today (run_uuid: " . ( $today[0]{run_uuid} // 'unknown' ) . ")" );
        }
        else {
            my $ug       = Data::UUID->new();
            my $run_uuid = lc( $ug->create_str() );

$run_logs->log_skipped_run(
                job_uuid  => $job_id,
                run_uuid  => $run_uuid,
                reason    => $reason,
                timestamp => $now,
            );

$logger->info( "Recorded skipped backup run for job " . ( $job_id // 'UNSET' ) . " (run_uuid: $run_uuid)" );
        }

1;
    } or $logger->warn( "Could not record the skipped backup run for job " . ( $job_id // 'job_id UNSET' ) . ": $@" );

return;
}

sub _execute_backup_job {
    my ( $jobs_manager, $job, $restart_lease_fh ) = @_;

my $job_id = $job->{job_id};

eval {
        # Generate a unique run_uuid for this backup run instance
        require Data::UUID;
        my $ug       = Data::UUID->new();
        my $run_uuid = lc( $ug->create_str() );

$logger->info( "Starting backup job: " . $job->{name} . " (job_id: $job_id, run_uuid: $run_uuid)" );

# Store the current_run_uuid in the backup_jobs table for tracking.
        # Job status and last_run_timestamp are the runner's to write (it sets
        # 'running' when it opens the run log and the final status plus
        # last_run_timestamp when it closes it); update_backup_job cannot write
        # either column, so the helper does not try.
        $jobs_manager->set_current_run_uuid( $job_id, $run_uuid );

# Fork the backup runner script in the background
        my $backup_runner_path = $Whostmgr::CometBackup::Constants::BACKUP_RUNNER_PATH;

unless ( -x $backup_runner_path ) {
            die "Backup runner script not found or not executable: $backup_runner_path";
        }

# Hand the sweep's shared agent-restart lease to the runner. Perl marks
        # everything it opens FD_CLOEXEC, so an inherited descriptor has to be
        # asked for, and it has to be asked for BEFORE the fork so a failure can
        # still be logged (the child's STDERR goes to the helper log a few lines
        # below, but it has no business making decisions). A failure is not
        # fatal: the run goes out either way and the lease then covers only up to
        # main()'s release.
        if ( $restart_lease_fh && !Whostmgr::CometBackup::Constants::agent_restart_lease_survives_exec($restart_lease_fh) ) {
            $logger->warn("Could not hand the agent-restart lease to the backup runner for job $job_id; this run is covered only until it is forked.");
        }

my $pid = fork();

if ( !defined $pid ) {

# Fork failed
            die "Failed to fork backup runner process: $!";
        }

if ( $pid == 0 ) {

# Child process - exec the backup runner.
            # All failure paths use POSIX::_exit so the child never unwinds
            # back into main()'s eval, which would corrupt the lock state.
            # umask 0027 was set at the top of main() and the child inherits it.
            open STDOUT, '>>', '/var/log/comet_backup_jobs_helper.log' or POSIX::_exit(1);
            open STDERR, '>>', '/var/log/comet_backup_jobs_helper.log' or POSIX::_exit(1);

# Become session leader to fully detach
            POSIX::setsid() or POSIX::_exit(1);

# Execute the backup runner with run_uuid; _exit if exec fails.
            # --bypass-schedule: the helper has already made its own scheduling
            # decision, so we skip the runner's redundant cooldown check. The
            # runner's "one runner at a time" guard still applies.
            #
            # THE INHERITED DESCRIPTOR IS DELIBERATE AND IS NOT A LEAK. The
            # sweep's shared agent-restart lease is passed through this exec on
            # purpose: the runner never reads it, but the kernel holds its
            # LOCK_SH for exactly as long as the runner lives, which is what
            # stops the TMPDIR auto-restart taking its LOCK_EX while this run is
            # between admission and its PID file. It is released by the runner
            # dying, however it dies. Do not "clean it up".
            exec( $backup_runner_path, '--bypass-schedule', '--job_uuid', $job_id, '--run_uuid', $run_uuid )
              or POSIX::_exit(1);
        }

# Parent process - just log that we started the runner
        $logger->info( "Forked backup runner process (PID: $pid) for job: " . $job->{name} );
    };

if ($@) {
        $logger->error( "Failed to execute backup job " . ( $job_id // 'job_id UNSET' ) . ": $@" );

# No runner was started, so drop the run_uuid recorded for it.
        eval { $jobs_manager->clear_current_run_uuid($job_id) };
    }
}

# CPANEL-54636: the once-per-calendar-day grouped-file maintenance pass.
# Reconciles all schedule/retention groups (idempotent -- catches jobs that were
# never re-run since the flag flipped) and purges retired legacy per-account
# homedir Protected Items whose retention window has expired. Guarded by a
# timestamp file so it runs at most once per calendar day.
sub _run_daily_grouped_file_maintenance {
    my ( $logger, $dry_run ) = @_;

my $guard = $Whostmgr::CometBackup::Constants::GROUPED_FILES_LAST_RECONCILE;

# Once per calendar day: skip if the guard was last stamped today.
    if ( -e $guard ) {
        my $mtime = ( stat $guard )[9] // 0;
        my @last  = localtime($mtime);
        my @now   = localtime( time() );
        return if $last[5] == $now[5] && $last[7] == $now[7];    # same year + day-of-year
    }

if ($dry_run) {
        print "Would run daily grouped-file reconcile + retired-PI purge\n";
        return;
    }

require Whostmgr::CometBackup::GroupedFiles;
    require Whostmgr::CometBackup::ProtectedItemMap;

my $summary = eval { Whostmgr::CometBackup::GroupedFiles::reconcile_all_groups() };
    if ( my $err = $@ ) {
        $logger->warn("Daily grouped-file reconcile failed: $err");
        return;    # do not stamp the guard, so the next tick retries
    }
    $logger->info(
        "Daily grouped-file reconcile complete",
        {
            groups       => $summary->{groups},
            pis_created  => $summary->{pis_created},
            pis_disabled => $summary->{pis_disabled},
            error_count  => scalar @{ $summary->{errors} || [] },
        }
    );

eval {
        _purge_retired_grouped_homedir_pis($logger);
        1;
    } or $logger->warn("Retired grouped-file homedir PI purge failed: $@");

# Stamp the guard only after a successful pass.
    if ( open my $fh, '>', $guard ) {
        print {$fh} time(), "\n";
        close $fh;
    }
    else {
        $logger->warn("Could not write grouped-files reconcile guard $guard: $!");
    }

return;
}

# Purge retired legacy per-account homedir Protected Items whose retention window
# has expired. Retention is per-GROUP: a CUSTOM-retention group's retired rows may
# be reclaimed after keep_days + a 7-day grace; a SMART-retention group's rows are
# NEVER auto-purged (documented). purge_retired_homedir_items_before_epoch takes a
# single cutoff, so to honor "smart -> no auto-purge" conservatively we skip the
# purge entirely whenever ANY active group uses smart retention, and otherwise use
# the LONGEST custom window across active groups as the cutoff (never reclaims a
# row a group might still want). Snapshots themselves expire via Comet-side
# retention regardless; this only prunes the local registry rows.
#
# CPANEL-55563 caveat: this cutoff bounds only the rows the per-group retirement
# writes. The flag-ON sweep (retire_unmanaged_legacy_homedir_items) writes the same
# marker on rows belonging to jobs OUTSIDE every active group -- inactive, or with
# home directories deselected -- whose retention is invisible here, so those rows
# can be reclaimed on a window derived from unrelated groups. Accepted residual;
# see that sub's POD before widening either side.
sub _purge_retired_grouped_homedir_pis {
    my ($logger) = @_;

my $map    = Whostmgr::CometBackup::ProtectedItemMap->new();
    my @groups = $map->get_active_file_backup_groups();
    return unless @groups;

require Cpanel::JSON;
    my $max_keep_days = 0;
    my $has_smart     = 0;
    for my $group (@groups) {
        my $retention = eval { Cpanel::JSON::Load( $group->{retention_canonical} // '{}' ) } || {};
        my $mode      = $retention->{mode} // 'smart';
        if ( $mode eq 'custom' ) {
            my $keep_days = int( $retention->{keep_days} // 0 );
            $max_keep_days = $keep_days if $keep_days > $max_keep_days;
        }
        else {
            $has_smart = 1;
        }
    }

# Smart retention -> no auto-purge. Any active smart group disables the purge
    # (conservative: never reclaim a row a smart-retention group might still want).
    return if $has_smart || $max_keep_days <= 0;

my $cutoff = time() - ( ( $max_keep_days + 7 ) * 86400 );               # +7-day grace
    my $purged = $map->purge_retired_homedir_items_before_epoch($cutoff);
    $logger->info(
        "Purged retired legacy homedir Protected Items past their retention window",
        { purged => $purged, cutoff_epoch => $cutoff, max_keep_days => $max_keep_days },
    ) if $purged;

return $purged;
}

# Run the main function if this script is executed directly
main() if !caller;

__END__

=head1 SEE ALSO

L<Whostmgr::CometBackup::BackupJobs>, L<Whostmgr::CometBackup::Scheduling>,
L<Whostmgr::CometBackup::DailyCoverage>, L<Whostmgr::CometBackup::AgentTmpdir>

=head1 AUTHOR

WebPros International, LLC

=head1 COPYRIGHT

=cut