Async jobs improvements

.github/workflows/ci.yml

@@ -81,6 +81,7 @@ jobs:
                   smoke/test_affinity_groups_projects
                   smoke/test_annotations
                   smoke/test_async_job
+                  smoke/test_async_jobs
                   smoke/test_attach_multiple_volumes
                   smoke/test_backup_recovery_dummy
                   smoke/test_certauthority_root

agent/src/main/java/com/cloud/agent/Agent.java

@@ -807,9 +807,7 @@ protected void processRequest(final Request request, final Link link) {
                         }
                         commandsInProgress.incrementAndGet();
                         try {
-                            if (cmd.isReconcile()) {
-                                cmd.setRequestSequence(request.getSequence());
-                            }
+                            cmd.setRequestSequence(request.getSequence());
                             answer = serverResource.executeRequest(cmd);
                         } finally {
                             commandsInProgress.decrementAndGet();
@@ -1103,9 +1101,7 @@ public void processOtherTask(final Task task) {
             Answer answer = null;
             commandsInProgress.incrementAndGet();
             try {
-                if (command.isReconcile()) {
-                    command.setRequestSequence(req.getSequence());
-                }
+                command.setRequestSequence(req.getSequence());
                 answer = serverResource.executeRequest(command);
             } finally {
                 commandsInProgress.decrementAndGet();

api/src/main/java/com/cloud/agent/api/Answer.java

@@ -21,6 +21,8 @@
 public class Answer extends Command {
     protected boolean result;
     protected String details;
+    // set when the command was stopped because its async job was cancelled
+    protected boolean cancelled;
 
     protected Answer() {
         this(null);
@@ -47,6 +49,20 @@ public String getDetails() {
         return details;
     }
 
+    public boolean isCancelled() {
+        return cancelled;
+    }
+
+    public void setCancelled(final boolean cancelled) {
+        this.cancelled = cancelled;
+    }
+
+    public static Answer createCancelledAnswer(final Command command, final String details) {
+        Answer answer = new Answer(command, false, details);
+        answer.setCancelled(true);
+        return answer;
+    }
+
     @Override
     public boolean executeInSequence() {
         return false;
@@ -69,6 +85,7 @@ public boolean equals(Object o) {
         Answer answer = (Answer) o;
 
         if (result != answer.result) return false;
+        if (cancelled != answer.cancelled) return false;
         if (details != null ? !details.equals(answer.details) : answer.details != null) return false;
 
         return true;
@@ -78,6 +95,7 @@ public boolean equals(Object o) {
     public int hashCode() {
         int result1 = super.hashCode();
         result1 = 31 * result1 + (result ? 1 : 0);
+        result1 = 31 * result1 + (cancelled ? 1 : 0);
         result1 = 31 * result1 + (details != null ? details.hashCode() : 0);
         return result1;
     }

api/src/main/java/com/cloud/event/EventTypes.java

@@ -918,6 +918,7 @@ public class EventTypes {
     public static final String EVENT_INSTANCE_BOOT_GROUP_READINESS_RULE_UPDATE = "INSTANCE.BOOT.GROUP.READINESS.RULE.UPDATE";
     public static final String EVENT_INSTANCE_BOOT_GROUP_READINESS_RULE_DELETE = "INSTANCE.BOOT.GROUP.READINESS.RULE.DELETE";
 
+    public static final String EVENT_JOB_CANCEL = "JOB.CANCEL";
 
     static {
 

api/src/main/java/com/cloud/exception/OperationCancelledException.java

@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.exception;
+
+import com.cloud.agent.api.Command;
+import com.cloud.utils.SerialVersionUID;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+import java.util.Arrays;
+import java.util.stream.Collectors;
+
+/** Thrown when an operation was stopped because its async job was cancelled. Unchecked: intermediate callers must not relabel it as a timeout. */
+public class OperationCancelledException extends CloudRuntimeException {
+
+    private static final long serialVersionUID = SerialVersionUID.OperationCancelledException;
+
+    long _agentId;
+    long _seqId;
+    long _elapsedSeconds;
+    boolean _isActive;
+    String _reason;
+
+    transient Command[] _cmds;
+
+    public OperationCancelledException(Command[] cmds, long agentId, long seqId, long elapsedSeconds, boolean isActive, String reason) {
+        super(commandNames(cmds) + " (seq " + seqId + ") on host " + agentId + " cancelled after " + elapsedSeconds + " s" + (reason != null ? ": " + reason : ""));
+        _agentId = agentId;
+        _seqId = seqId;
+        _elapsedSeconds = elapsedSeconds;
+        _cmds = cmds;
+        _isActive = isActive;
+        _reason = reason;
+    }
+
+    private OperationCancelledException(String message, Command[] cmds, long agentId, String reason) {
+        super(message);
+        _agentId = agentId;
+        _cmds = cmds;
+        _reason = reason;
+    }
+
+    public OperationCancelledException(String reason) {
+        super(reason);
+        _reason = reason;
+    }
+
+    /** The commands were never sent: its job had already been cancelled. */
+    public static OperationCancelledException updateCmdsNotSent(Command[] cmds, long agentId, long jobId) {
+        final String reason = "job-" + jobId + " was cancelled";
+        return new OperationCancelledException(commandNames(cmds) + " to host " + agentId + " not sent: " + reason, cmds, agentId, reason);
+    }
+
+    static String commandNames(Command[] cmds) {
+        if (cmds == null || cmds.length == 0) {
+            return "Command";
+        }
+        return Arrays.stream(cmds).map(cmd -> cmd == null ? "null" : cmd.getClass().getSimpleName()).collect(Collectors.joining(", "));
+    }
+
+    public long getAgentId() {
+        return _agentId;
+    }
+
+    public long getSequenceId() {
+        return _seqId;
+    }
+
+    public long getElapsedSeconds() {
+        return _elapsedSeconds;
+    }
+
+    public boolean isActive() {
+        return _isActive;
+    }
+
+    public String getReason() {
+        return _reason;
+    }
+
+    public Command[] getCommands() {
+        return _cmds;
+    }
+}

api/src/main/java/com/cloud/vm/VirtualMachine.java

@@ -105,6 +105,7 @@ public static StateMachine2<State, VirtualMachine.Event, VirtualMachine> getStat
             s_fsm.addTransition(new Transition<State, Event>(State.Starting, VirtualMachine.Event.AgentReportRunning, State.Running, Arrays.asList(new Impact[]{Impact.USAGE})));
             s_fsm.addTransition(new Transition<State, Event>(State.Starting, VirtualMachine.Event.AgentReportStopped, State.Stopped, null));
             s_fsm.addTransition(new Transition<State, Event>(State.Starting, VirtualMachine.Event.AgentReportShutdowned, State.Stopped, null));
+            s_fsm.addTransition(new Transition<State, Event>(State.Starting, VirtualMachine.Event.StopRequested, State.Stopping, null));
             s_fsm.addTransition(new Transition<State, Event>(State.Destroyed, VirtualMachine.Event.RecoveryRequested, State.Stopped, Arrays.asList(new Impact[]{Impact.USAGE})));
             s_fsm.addTransition(new Transition<State, Event>(State.Destroyed, VirtualMachine.Event.ExpungeOperation, State.Expunging, null));
             s_fsm.addTransition(new Transition<State, Event>(State.Running, VirtualMachine.Event.MigrationRequested, State.Migrating, null));

api/src/main/java/org/apache/cloudstack/api/APICommand.java

@@ -52,4 +52,6 @@
     Class<?>[] entityType() default {};
 
     String httpMethod() default "";
+
+    boolean cancellable() default false;
 }

api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java

@@ -94,7 +94,8 @@ public enum ApiCommandResourceType {
     KmsKey(org.apache.cloudstack.kms.KMSKey.class),
     HsmProfile(org.apache.cloudstack.kms.HSMProfile.class),
     InstanceBootGroup(org.apache.cloudstack.vm.bootgroup.InstanceBootGroup.class),
-    InstanceBootGroupReadinessRule(InstanceBootGroupReadinessRule.class);
+    InstanceBootGroupReadinessRule(InstanceBootGroupReadinessRule.class),
+    Job(org.apache.cloudstack.jobs.JobInfo.class);
 
     private final Class<?> clazz;
 

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -716,6 +716,7 @@ public class ApiConstants {
     public static final String USAGE_ID = "usageid";
     public static final String USAGE_NAME = "usagename";
     public static final String USAGE_TYPE = "usagetype";
+    public static final String USAGE_SERVER = "usageserver";
     public static final String INCLUDE_TAGS = "includetags";
 
     public static final String VLAN = "vlan";

api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java

@@ -22,6 +22,7 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.cloudstack.api.command.user.job.CancelAsyncJobCmd;
 import org.apache.cloudstack.api.response.ConsoleSessionResponse;
 import org.apache.cloudstack.consoleproxy.ConsoleSession;
 import org.apache.cloudstack.acl.apikeypair.ApiKeyPair;
@@ -397,6 +398,8 @@ TemplatePermissionsResponse createTemplatePermissionsResponse(ResponseView view,
 
     AsyncJobResponse queryJobResult(QueryAsyncJobResultCmd cmd);
 
+    AsyncJobResponse cancelJobResponse(CancelAsyncJobCmd cmd);
+
     NetworkOfferingResponse createNetworkOfferingResponse(NetworkOffering offering);
 
     NetworkResponse createNetworkResponse(ResponseView view, Network network);

api/src/main/java/org/apache/cloudstack/api/command/admin/usage/ListUsageJobsCmd.java

@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.usage;
+
+import java.util.Date;
+
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseListCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.response.ListResponse;
+import org.apache.cloudstack.api.response.UsageJobResponse;
+
+@APICommand(name = "listUsageJobs", description = "Lists the usage jobs.", responseObject = UsageJobResponse.class,
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "24.0")
+public class ListUsageJobsCmd extends BaseListCmd {
+
+    /////////////////////////////////////////////////////
+    //////////////// API parameters /////////////////////
+    /////////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "The start date from which the usage jobs should be listed. Only jobs started on or after this date will be included. (use format \"yyyy-MM-dd'T'HH:mm:ss'+'SSSS\")")
+    private Date startDate;
+
+    @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "The end date up to which the usage jobs should be listed. Only jobs started on or before this date will be included. (use format \"yyyy-MM-dd'T'HH:mm:ss'+'SSSS\")")
+    private Date endDate;
+
+    @Parameter(name = ApiConstants.USAGE_SERVER, type = CommandType.STRING, description = "The usage server host name or ip")
+    private String usageServer;
+
+    @Parameter(name = ApiConstants.DURATION, type = CommandType.INTEGER, description = "the duration in hours to list the usage jobs started or completed within that period up to now.")
+    private Integer duration;
+
+    /////////////////////////////////////////////////////
+    /////////////////// Accessors ///////////////////////
+    /////////////////////////////////////////////////////
+
+    public Date getStartDate() {
+        return startDate;
+    }
+
+    public Date getEndDate() {
+        return endDate;
+    }
+
+    public String getUsageServer() {
+        return usageServer;
+    }
+
+    public Integer getDuration() {
+        return duration;
+    }
+
+    /////////////////////////////////////////////////////
+    /////////////// API Implementation///////////////////
+    /////////////////////////////////////////////////////
+    @Override
+    public void execute() {
+        ListResponse<UsageJobResponse> response = _usageService.getUsageJobs(this);
+        response.setResponseName(getCommandName());
+        this.setResponseObject(response);
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DestroyVMCmdByAdmin.java

@@ -28,7 +28,8 @@
 
 @APICommand(name = "destroyVirtualMachine", description = "Destroys  an Instance. Once destroyed, only the administrator can recover it.", responseObject = UserVmResponse.class, responseView = ResponseView.Full, entityType = {VirtualMachine.class},
         requestHasSensitiveInfo = false,
-        responseHasSensitiveInfo = true)
+        responseHasSensitiveInfo = true,
+        cancellable = true)
 public class DestroyVMCmdByAdmin extends DestroyVMCmd implements AdminCmd {
 
     @Parameter( name = ApiConstants.FORCED,

api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVMCmd.java

@@ -48,10 +48,10 @@
         description = "Attempts Migration of an Instance to a different host or Root volume of the Instance to a different storage pool",
         responseObject = UserVmResponse.class, entityType = {VirtualMachine.class},
         requestHasSensitiveInfo = false,
-        responseHasSensitiveInfo = true)
+        responseHasSensitiveInfo = true,
+        cancellable = true)
 public class MigrateVMCmd extends BaseAsyncCmd {
 
-
     /////////////////////////////////////////////////////
     //////////////// API parameters /////////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVirtualMachineWithVolumeCmd.java

@@ -49,9 +49,9 @@
             description = "Attempts Migration of an Instance with its volumes to a different host",
             responseObject = UserVmResponse.class, entityType = {VirtualMachine.class},
             requestHasSensitiveInfo = false,
-            responseHasSensitiveInfo = true)
-public class MigrateVirtualMachineWithVolumeCmd extends BaseAsyncCmd {
-
+            responseHasSensitiveInfo = true,
+            cancellable = true)
+public class MigrateVirtualMachineWithVolumeCmd extends BaseAsyncCmd  {
 
     /////////////////////////////////////////////////////
     //////////////// API parameters /////////////////////

api/src/main/java/org/apache/cloudstack/api/command/admin/vm/RebootVMCmdByAdmin.java

@@ -25,5 +25,5 @@
 import com.cloud.vm.VirtualMachine;
 
 @APICommand(name = "rebootVirtualMachine", description = "Reboots  an Instance.", responseObject = UserVmResponse.class, responseView = ResponseView.Full, entityType = {VirtualMachine.class},
-        requestHasSensitiveInfo = false, responseHasSensitiveInfo = true)
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, cancellable = true)
 public class RebootVMCmdByAdmin extends RebootVMCmd implements AdminCmd {}

api/src/main/java/org/apache/cloudstack/api/command/admin/vm/StartVMCmdByAdmin.java

@@ -25,5 +25,5 @@
 import com.cloud.vm.VirtualMachine;
 
 @APICommand(name = "startVirtualMachine", responseObject = UserVmResponse.class, description = "Starts  an Instance.", responseView = ResponseView.Full, entityType = {VirtualMachine.class},
-        requestHasSensitiveInfo = false, responseHasSensitiveInfo = true)
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, cancellable = true)
 public class StartVMCmdByAdmin extends StartVMCmd implements AdminCmd {}

api/src/main/java/org/apache/cloudstack/api/command/admin/vm/StopVMCmdByAdmin.java

@@ -25,5 +25,5 @@
 import com.cloud.vm.VirtualMachine;
 
 @APICommand(name = "stopVirtualMachine", responseObject = UserVmResponse.class, description = "Stops  an Instance.", responseView = ResponseView.Full, entityType = {VirtualMachine.class},
-        requestHasSensitiveInfo = false, responseHasSensitiveInfo = true)
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, cancellable = true)
 public class StopVMCmdByAdmin extends StopVMCmd implements AdminCmd {}

api/src/main/java/org/apache/cloudstack/api/command/admin/vmsnapshot/RevertToVMSnapshotCmdByAdmin.java

@@ -23,5 +23,5 @@
 import org.apache.cloudstack.api.response.UserVmResponse;
 
 @APICommand(name = "revertToVMSnapshot", description = "Revert Instance from a vmsnapshot.", responseObject = UserVmResponse.class, since = "4.2.0", responseView = ResponseView.Full,
-        requestHasSensitiveInfo = false, responseHasSensitiveInfo = true)
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, cancellable = true)
 public class RevertToVMSnapshotCmdByAdmin extends RevertToVMSnapshotCmd implements AdminCmd {}

api/src/main/java/org/apache/cloudstack/api/command/admin/volume/AttachVolumeCmdByAdmin.java

@@ -25,5 +25,5 @@
 import com.cloud.vm.VirtualMachine;
 
 @APICommand(name = "attachVolume", description = "Attaches a disk volume to  an Instance.", responseObject = VolumeResponse.class, responseView = ResponseView.Full, entityType = {VirtualMachine.class},
-        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, cancellable = true)
 public class AttachVolumeCmdByAdmin extends AttachVolumeCmd implements AdminCmd {}

api/src/main/java/org/apache/cloudstack/api/command/admin/volume/CreateVolumeCmdByAdmin.java

@@ -27,5 +27,5 @@
 
 @APICommand(name = "createVolume", responseObject = VolumeResponse.class, description = "Creates a disk volume from a disk offering. This disk volume must still be attached to  an Instance to make use of it.", responseView = ResponseView.Full, entityType = {
         Volume.class, VirtualMachine.class},
-        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, cancellable = true)
 public class CreateVolumeCmdByAdmin extends CreateVolumeCmd implements AdminCmd {}

api/src/main/java/org/apache/cloudstack/api/command/admin/volume/DestroyVolumeCmdByAdmin.java

@@ -33,7 +33,8 @@
             since = "4.14.0",
             authorized = {RoleType.Admin},
             requestHasSensitiveInfo = false,
-            responseHasSensitiveInfo = true)
+            responseHasSensitiveInfo = true,
+            cancellable = true)
 public class DestroyVolumeCmdByAdmin extends DestroyVolumeCmd implements AdminCmd {