Merge discvr-26.7 to develop

Author: bbimber

.gitignore

@@ -22,4 +22,7 @@ SequenceAnalysis/resources/credits/jars.txt
 SequenceAnalysis/resources/credits/dependencies.txt
 
 OpenLdapSync/resources/credits/dependencies.txt
-OpenLdapSync/resources/credits/jars.txt
+OpenLdapSync/resources/credits/jars.txt
+
+cluster/resources/credits/jars.txt
+cluster/resources/credits/dependencies.txt

OpenLdapSync/src/org/labkey/openldapsync/ldap/LdapSyncRunner.java

@@ -269,16 +269,16 @@ private void setUserActive(User u, boolean active, String reason)
         try
         {
             log("Changing active state of user: " + u.getEmail() + " to " + active + (reason == null ? "" : ", reason: " + reason));
-            _usersInactivated++;
 
             if (!_previewOnly)
             {
                 UserManager.setUserActive(_settings.getLabKeyAdminUser(), u, active);
             }
+            _usersInactivated++;
         }
         catch (SecurityManager.UserManagementException e)
         {
-            _log.error("Unable to deactive user: " + u.getEmail());
+            _log.error("Unable to deactivate user: " + u.getEmail(), e);
         }
     }
 

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/SequenceOutputFile.java

@@ -16,6 +16,7 @@
 package org.labkey.api.sequenceanalysis;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.logging.log4j.Logger;
 import org.json.JSONObject;
 import org.labkey.api.data.Container;
 import org.labkey.api.data.ContainerManager;
@@ -24,6 +25,11 @@
 import org.labkey.api.exp.api.ExpData;
 import org.labkey.api.exp.api.ExperimentService;
 import org.labkey.api.pipeline.PipelineJobService;
+import org.labkey.api.security.User;
+import org.labkey.api.security.permissions.Permission;
+import org.labkey.api.security.permissions.ReadPermission;
+import org.labkey.api.util.logging.LogHelper;
+import org.labkey.api.view.UnauthorizedException;
 
 import java.io.File;
 import java.io.Serializable;
@@ -34,6 +40,8 @@
  */
 public class SequenceOutputFile implements Serializable
 {
+    private static final Logger _log = LogHelper.getLogger(SequenceOutputFile.class, "Messages related to SequenceOutputFile");
+
     private Integer _rowid;
     private String _name;
     private String _description;
@@ -211,6 +219,28 @@ public void setModified(Date modified)
         _modified = modified;
     }
 
+    public static SequenceOutputFile getForId(Integer rowId, User u)
+    {
+        return getForId(rowId, u, ReadPermission.class);
+    }
+
+    public static SequenceOutputFile getForId(Integer rowId, User u, Class<? extends Permission> perm)
+    {
+        SequenceOutputFile so = getForId(rowId);
+        if (so.getContainerObj() == null)
+        {
+            _log.error("SequenceOutputFile lacks a valid container: " + rowId);
+            return null;
+        }
+
+        if (!so.getContainerObj().hasPermission(u, perm))
+        {
+            throw new UnauthorizedException("Insufficient permissions: " + rowId);
+        }
+
+        return so;
+    }
+
     public static SequenceOutputFile getForId(Integer rowId)
     {
         if (PipelineJobService.get().getLocationType() != PipelineJobService.LocationType.WebServer)

SequenceAnalysis/resources/queries/sequenceanalysis/distinctAnalysisSets.sql

@@ -0,0 +1,3 @@
+SELECT
+    DISTINCT rowid, name
+FROM sequenceanalysis.analysisSets

SequenceAnalysis/resources/web/SequenceAnalysis/window/AddFileSetsWindow.js

@@ -50,7 +50,8 @@ Ext4.define('SequenceAnalysis.window.AddFileSetsWindow', {
                         type: 'labkey-store',
                         containerPath: Laboratory.Utils.getQueryContainerPath(),
                         schemaName: 'laboratory',
-                        sql: 'SELECT DISTINCT rowid, name FROM sequenceanalysis.analysisSets',
+                        queryName: 'distinctAnalysisSets',
+                        columns: 'rowid,name',
                         autoLoad: true
                     },
                     valueField: 'rowid',

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java

@@ -44,6 +44,7 @@
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -81,25 +82,32 @@ public String getName()
         return "DeleteSequenceAnalysisArtifacts";
     }
 
+    private int _jobId = -1;
+
     // NOTE: if there is a more direct way to locate the JobID this hack should be replaced
     private void checkJobCancelled(Logger log)
     {
-        // Make the assumption there is only one active maintenance job at a time:
-        SimpleFilter filter = new SimpleFilter(FieldKey.fromString("description"), SYSTEM_MAINTENANCE_DESCRIPTION).
-                addCondition(FieldKey.fromString("container"), ContainerManager.getRoot().getId()).
-                addCondition(FieldKey.fromString("modified"), new Date(), CompareType.DATE_EQUAL);
-        int rowId = new TableSelector(DbSchema.get("pipeline", DbSchemaType.Module).getTable(JOB_TABLE), PageFlowUtil.set("RowId", "Status"), filter, null).getMapCollection().stream().filter(map -> {
-            String val = String.valueOf(map.get("status"));
-            return val != null && (val.toLowerCase().startsWith(PipelineJob.TaskStatus.cancelling.name()) || val.toLowerCase().startsWith(PipelineJob.TaskStatus.running.name()));
-        }).map(rs -> Integer.parseInt(String.valueOf(rs.get("rowid")))).max(Integer::compareTo).orElse(-1);
-
-        if (rowId == -1)
+        if (_jobId == -1)
         {
-            log.warn("Unable to find rowId for job", new Exception("Unable to find rowId for job"));
-            return;
+            // Make the assumption there is only one active maintenance job at a time:
+            SimpleFilter filter = new SimpleFilter(FieldKey.fromString("description"), SYSTEM_MAINTENANCE_DESCRIPTION).
+                    addCondition(FieldKey.fromString("container"), ContainerManager.getRoot().getId()).
+                    addCondition(FieldKey.fromString("modified"), LocalDate.now().minusDays(2), CompareType.DATE_GTE);
+            int rowId = new TableSelector(DbSchema.get("pipeline", DbSchemaType.Module).getTable(JOB_TABLE), PageFlowUtil.set("RowId", "Status"), filter, null).getMapCollection().stream().filter(map -> {
+                String val = String.valueOf(map.get("status"));
+                return val != null && (val.toLowerCase().startsWith(PipelineJob.TaskStatus.cancelling.name()) || val.toLowerCase().startsWith(PipelineJob.TaskStatus.running.name()));
+            }).map(rs -> Integer.parseInt(String.valueOf(rs.get("rowid")))).max(Integer::compareTo).orElse(-1);
+
+            if (rowId == -1)
+            {
+                log.warn("Unable to find rowId for job", new Exception("Unable to find rowId for job"));
+                return;
+            }
+
+            _jobId = rowId;
         }
 
-        PipelineStatusFile sf = PipelineService.get().getStatusFile(rowId);
+        PipelineStatusFile sf = PipelineService.get().getStatusFile(_jobId);
         if (PipelineJob.TaskStatus.cancelling.name().equalsIgnoreCase(sf.getStatus()))
         {
             throw new CancelledException();

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java

@@ -90,6 +90,12 @@ public boolean doSplitJobs()
         return true;
     }
 
+    @Override
+    public boolean supportsSraArchivedData()
+    {
+        return true;
+    }
+
     public class Processor implements SequenceOutputProcessor
     {
         @Override
@@ -99,7 +105,7 @@ public void init(JobContext ctx, List<SequenceOutputFile> inputFiles, List<Recor
             {
                 if (so.getReadset() != null)
                 {
-                    ctx.getSequenceSupport().cacheReadset(so.getReadset(), ctx.getJob().getUser());
+                    ctx.getSequenceSupport().cacheReadset(so.getReadset(), ctx.getJob().getUser(), true);
                 }
                 else
                 {
@@ -148,7 +154,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
                     args.addAll(extraArgs);
                 }
 
-                File output = new File(ctx.getWorkingDirectory(), FileUtil.getBaseName(input) + ".txt");
+                File output = FileUtil.appendName(ctx.getWorkingDirectory(), FileUtil.getBaseName(input) + ".txt");
                 Wrapper wrapper = new Wrapper(ctx.getLogger());
                 wrapper.execute(input, ctx.getSequenceSupport().getCachedGenome(so.getLibrary_id()).getWorkingFastaFile(), output, args);
 

SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/PrintReadBackedHaplotypesHandler.java

@@ -132,7 +132,7 @@ private List<AnalysisModel> parseAndCreateReadsets() throws PipelineJobException
                         }
                     }
 
-                    getPipelineJob().getSequenceSupport().cacheGenome(SequenceAnalysisService.get().getReferenceGenome(o.getInt("library_id"), getJob().getUser()));
+                    getPipelineJob().getSequenceSupport().cacheGenome(SequenceAnalysisService.get().getReferenceGenome(o.getInt("library_id"), getJob().getUser()), true);
                     getPipelineJob().getSequenceSupport().cacheReadset(r);
                 }
             }

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentImportInitTask.java

@@ -156,7 +156,7 @@ else if (steps.size() > 1)
                     throw new PipelineJobException("Reference file does not exist: " + refFasta.getPath());
                 }
 
-                getPipelineJob().getSequenceSupport().cacheGenome(output.getReferenceGenome());
+                getPipelineJob().getSequenceSupport().cacheGenome(output.getReferenceGenome(), true);
 
                 getHelper().getFileManager().addStepOutputs(action, output);
                 getHelper().getFileManager().cleanup(Collections.singleton(action));