workflow/cleanup_taskmanagers.csh at main · JSOC-SDP/workflow · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
#! /bin/csh -f

if ( ! $?WORKFLOW_DATA ) then
    echo WORKFLOW_DATA environment variable is undefined
    exit 1
endif

set noglob

set TMP = /tmp/$$
rm -f $TMP
rm -f $TMP.t

# wait for gatekeeper idle
cd $$WORKFLOW_DATA
while (-e GATEKEEPERBUSY)
    sleep 1
    echo -n '.'
end
echo " "

if (-e restart.log) then
    grep '^On' restart.log >$TMP
    set nrestart = `wc -l <$TMP`

    if ($nrestart > 0) then
        set On_line = `head -1 $TMP`
        set gatekeeper_host = $On_line[2]
        set now_host = `hostname`
        if ($gatekeeper_host != $now_host) then
            echo Gatekeeper is running on $gatekeeper_host, NOT $now_host.
            echo exit.
            exit 1
        endif
    endif

    echo No hostname in restart.log
endif


ps ax | grep manager >$TMP.t
grep ticket <$TMP.t >$TMP
set n_managers = `wc -l <$TMP`

# Now $TMP should contain a list of active taskmanagers

set i_manager = 1
while ($i_manager <= $n_managers)
    set prog = `head --lines $i_manager <$TMP | tail -1`
    set thisPID = $prog[1]
    set task = NOTFOUND
    set gate = NOTFOUND
    set ticket = NOTFOUND
    set nargs = $#prog
    set iarg = 8

    while ($iarg <= $nargs)
        set thisarg = $prog[$iarg]
        if ($thisarg =~ "task=*") then
        set task = $thisarg:s/task=//
        else if ($thisarg =~ gate=*) then
        set gate = $thisarg:s/gate=//
        else if ($thisarg =~ ticket=*) then
        set ticket = $thisarg:s/ticket=//
        endif
        @ iarg = $iarg + 1
    end

    if ($task != NOTFOUND && $gate != NOTFOUND && $ticket != NOTFOUND) then
    # Now have a pipeline taskmanager process that is running, check for validity
    # there should be a manager.pid for this ticket in the active logs dir
        if (-e $WORKFLOW_DATA/tasks/$task/logs/$ticket/manager.pid) then
            set foundPID = `cat $WORKFLOW_DATA/tasks/$task/logs/$ticket/manager.pid`
            if ($thisPID == $foundPID) then
                echo $thisPID is still active
            else
                echo $thisPID does not match given tasks PID: $foundPID, so ignore
            endif
        else
            echo $thisPID does not have an active task
            if (-e $WORKFLOW_DATA/tasks/$task/archive/logs/$ticket) then
                echo "       " this prog is for a completed task, ticket found $$WORKFLOW_DATA/tasks/$task/archive/logs/$ticket
                echo "       " kill $thisPID for gate=$gate task=$task ticket=$ticket
                kill -9 $thisPID
            else
                echo "       " this PID is apparently a leftover, but needs to be examined.  do nothing.
            endif
        endif
    else
        echo $thisPID does not have a valid gate, task, or ticket, so ignore
    endif
    @ i_manager = $i_manager + 1
end

rm -f $TMP
rm -f $TMP.t