diff --git a/ai.go b/ai.go index fb944e68..b7da6253 100644 --- a/ai.go +++ b/ai.go @@ -8,11 +8,11 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" - "unicode/utf8" "errors" "fmt" "io/ioutil" "log" + "math" "math/rand" "net/http" "net/url" @@ -24,7 +24,8 @@ import ( "strings" "sync" "time" - "math" + "unicode/utf8" + openai "github.com/sashabaranov/go-openai" uuid "github.com/satori/go.uuid" "google.golang.org/api/customsearch/v1" @@ -45,14 +46,15 @@ var standalone bool // var model = "gpt-5-mini" var model = "gpt-5-mini" + //var model = "gpt-5.4-nano" //var model = "gpt-5.2-codex" var fallbackModel = "" var assistantId = os.Getenv("OPENAI_ASSISTANT_ID") var docsVectorStoreID = os.Getenv("OPENAI_DOCS_VS_ID") -var skipAgentWait = os.Getenv("SHUFFLE_SKIP_AGENT_WAIT") -var agentRunLocation = os.Getenv("SHUFFLE_AGENT_RUN_LOCATION") +var skipAgentWait = os.Getenv("SHUFFLE_SKIP_AGENT_WAIT") +var agentRunLocation = os.Getenv("SHUFFLE_AGENT_RUN_LOCATION") var assistantModel = model var decisionParameterName = "shuffle_agent_decision_id" @@ -67,7 +69,7 @@ func init() { } reasoningEffort := os.Getenv("AI_REASONING_EFFORT") - if reasoningEffort == "minimal" || reasoningEffort == "low" || reasoningEffort == "medium" || reasoningEffort == "high" { + if reasoningEffort == "minimal" || reasoningEffort == "low" || reasoningEffort == "medium" || reasoningEffort == "high" { aiReasoningEffort = reasoningEffort } @@ -76,13 +78,13 @@ func init() { } func EstimatePromptTokens(messages []openai.ChatCompletionMessage) int64 { - totalChars := 0 - for _, msg := range messages { - totalChars += utf8.RuneCountInString(msg.Content) - totalChars += 20 - } - - return int64((totalChars + 3) / 4) + totalChars := 0 + for _, msg := range messages { + totalChars += utf8.RuneCountInString(msg.Content) + totalChars += 20 + } + + return int64((totalChars + 3) / 4) } // Provide an incident triage and response plan for the reported incident finding. Make a short list of actions to perform in the following format: [{"title": "Title of the task", "category": "triage/containment/recovery/communication/documentation", "completed": false, "createdBy": "ai-agent@shuffler.io"}]. ONLY output as JSON array and nothing more. After the list is made, add these to the metadata.extensions.custom_attributes.tasks[] in the next action. @@ -413,7 +415,7 @@ func FindHttpBody(fullBody []byte) (HTTPOutput, []byte, error) { // Make result into a body as well err = json.Unmarshal([]byte(kmsResponse.Result), httpOutput) if err != nil { - if len(kmsResponse.Result) > 0 { + if len(kmsResponse.Result) > 0 { log.Printf("[ERROR] Failed to unmarshal Schemaless HTTP Output response (2): %s. Data: %s", err, kmsResponse.Result) } @@ -1642,9 +1644,9 @@ func extractDecisionArray(rawText string) ([]AgentDecision, error) { stringReader := strings.NewReader(rawText[byteIndex:]) jsonDecoder := json.NewDecoder(stringReader) decodeErr := jsonDecoder.Decode(&decodedRawDecisions) - + if decodeErr != nil || len(decodedRawDecisions) == 0 { - continue + continue } // Check if the first item has an "action" key @@ -1657,7 +1659,7 @@ func extractDecisionArray(rawText string) ([]AgentDecision, error) { var fields []rawField if unmarshalErr := json.Unmarshal(rawFields, &fields); unmarshalErr == nil { normalizeRawDecisionFields(fields) - + fixedFieldsBytes, marshalErr := json.Marshal(fields) if marshalErr == nil { decodedRawDecisions[mapIndex]["fields"] = fixedFieldsBytes @@ -1670,13 +1672,13 @@ func extractDecisionArray(rawText string) ([]AgentDecision, error) { if marshalErr != nil { continue } - + var finalDecisions []AgentDecision structUnmarshalErr := json.Unmarshal(marshaledJSONBytes, &finalDecisions) if structUnmarshalErr != nil { continue } - + return finalDecisions, nil } @@ -1702,17 +1704,17 @@ func extractDecisionJSONL(rawText string) ([]AgentDecision, error) { bytesConsumedByDecoder := int(jsonDecoder.InputOffset()) if bytesConsumedByDecoder <= 0 { - byteIndex++ + byteIndex++ } else { byteIndex += bytesConsumedByDecoder } if decodeErr != nil { - continue + continue } if _, hasAction := rawMap["action"]; !hasAction { - continue + continue } // Fix the "fields" array if it exists @@ -1914,7 +1916,7 @@ func AutofixAppLabels(ctx context.Context, app WorkflowApp, label string, keys [ } // Update the actual app? - if debug { + if debug { log.Printf("[DEBUG] UPDATEINDEX CATEGORY (%s): %#v", app.Name, updatedIndex) } } @@ -1956,7 +1958,7 @@ func AutofixAppLabels(ctx context.Context, app WorkflowApp, label string, keys [ actionStruct.Action = string(guessedActionString) } } else { - if debug { + if debug { //log.Printf("[DEBUG] Failed to get app from cache in AutofixAppLabels for app %s (%s): %s", app.Name, app.ID, cacheGeterr) } } @@ -2190,7 +2192,7 @@ Do not add explanations, comments, or extra formatting. Only return valid JSON.` // FIXME: Add the label to the OpenAPI action as well? // 0x0elliot: Would we want to do this through an API on standalone? - if debug { + if debug { log.Printf("[DEBUG] App: %#v, standalone: %v, updatedIndex: %d, len(app.Actions): %d", app.Name, standalone, updatedIndex, len(app.Actions)) } @@ -2307,12 +2309,12 @@ Do not add explanations, comments, or extra formatting. Only return valid JSON.` } func GetActionAIResponse(ctx context.Context, resp http.ResponseWriter, user User, org Org, outputFormat string, input QueryInput) ([]byte, error) { - if len(org.Id) == 0 { + if len(org.Id) == 0 { if len(input.OrgId) > 0 && user.ActiveOrg.Id == "" { user.ActiveOrg.Id = input.OrgId } - if len(user.ActiveOrg.Id) > 0 { + if len(user.ActiveOrg.Id) > 0 { newOrg, err := GetOrg(ctx, user.ActiveOrg.Id) if err != nil { log.Printf("[ERROR] Failed to load orgid '%s' in ai response check", user.ActiveOrg.Id) @@ -2332,7 +2334,7 @@ func GetActionAIResponse(ctx context.Context, resp http.ResponseWriter, user Use if project.Environment == "cloud" && !user.SupportAccess { //if org.SyncFeatures.ShuffleGPT.Active && org.SyncFeatures.ShuffleGPT.Usage < org.SyncFeatures.ShuffleGPT.Limit { - // Most should never reach this + // Most should never reach this if org.SyncFeatures.ShuffleGPT.Usage < 1000 { log.Printf("[AUDIT] Org %#v (%s) has access to the auto feature. Allowing user %s to use it", org.Name, org.Id, user.Username) org.SyncFeatures.ShuffleGPT.Usage += 1 @@ -2575,7 +2577,7 @@ func GetActionAIResponse(ctx context.Context, resp http.ResponseWriter, user Use appname = appname1.(string) } - if debug { + if debug { log.Printf("[DEBUG] Starting AI Translation with app '%s' and category '%s' for query '%s'", appname, category, inputQuery) } @@ -7179,7 +7181,7 @@ func RunAgentFinishVerifier(ctx context.Context, orgId string, executionId strin } } - verifierSystem := `You are a completion verifier for an AI agent system. Reply with ONLY valid JSON, no markdown: + verifierSystem := `You are a completion verifier for an AI agent system. Reply with ONLY valid JSON, no markdown: {"pass": true, "reason": "one sentence"} or {"pass": false, "reason": "one sentence"} Rules: @@ -7247,7 +7249,7 @@ func abortAgentExecution(ctx context.Context, execution WorkflowExecution, start // FIXME: Where do we find original_input? // What if it doesn't exist? - if len(agentOutput.OriginalInput) == 0 { + if len(agentOutput.OriginalInput) == 0 { } if !lastDecisionIsFinish { @@ -7357,26 +7359,34 @@ func sendAITokenLimitAlert(ctx context.Context, execution WorkflowExecution, ful appRunsLimit := int64(0) orgStats, statsErr := GetOrgStatistics(ctx, billingOrgId) if statsErr == nil && orgStats != nil { - totalAppExecutions = orgStats.MonthlyAppExecutions + orgStats.MonthlyChildAppExecutions + stats := GetCorrectedStats(orgStats) + totalAppExecutions = stats.MonthlyAppExecutions + stats.MonthlyChildAppExecutions } if fullOrg != nil { appRunsLimit = fullOrg.SyncFeatures.AppExecutions.Limit } + appRunsUsagePercentage := float64(totalAppExecutions) / float64(appRunsLimit) * 100 subjectLine := fmt.Sprintf("%d%% of your AI token limit", int64(aiPercentage)) Subject := fmt.Sprintf("[Shuffle]: You've reached %s for your tenant %s", subjectLine, orgName) AiRecommendation := "Tip: Connect your own AI provider app to use your own keys and bypass the AI token limit entirely." + + if tokenLimit == 0 { + tokenLimit = 10000000 + } + substitutions := map[string]interface{}{ - "app_runs_usage": totalAppExecutions, - "app_runs_limit": appRunsLimit, - "subject_string": subjectLine, - "ai_tokens_usage": monthlyTokensUsed, - "ai_tokens_limit": tokenLimit, - "org_name": orgName, - "org_id": billingOrgId, - "admin_email": orgName, - "app_runs_usage_percentage": int64(aiPercentage), - "ai_recommendation": AiRecommendation, + "app_runs_usage": totalAppExecutions, + "app_runs_limit": appRunsLimit, + "subject_string": subjectLine, + "ai_tokens_usage": monthlyTokensUsed, + "ai_tokens_limit": tokenLimit, + "org_name": orgName, + "org_id": billingOrgId, + "admin_email": orgName, + "app_runs_usage_percentage": int64(appRunsUsagePercentage), + "ai_tokens_usage_percentage": int64(aiPercentage), + "ai_recommendation": AiRecommendation, } err = sendMailSendgridV2( @@ -7817,13 +7827,12 @@ CRITICAL RULES FOR THE AGENT return systemRule, templateContext, nil } - func getWorkflowEditPromptRemovals() []string { return []string{ ` - **Destructive Guard:** - If action is DESTRUCTIVE (stop/delete/remove) -> Set "approval_required": true on the action/tool.`, - `// true IF the action seems risky or destructive and requires user approval. Otherwise false`, - `### DATA REDUCTION: + `// true IF the action seems risky or destructive and requires user approval. Otherwise false`, + `### DATA REDUCTION: data_filter: - "full": The default value of the data_filter is full. Use for all non-data-returning calls or when you need the entire response. - "list": Use for ALL data calls. Request ONLY essential fields. If the schema is completely unknown, fallback to "full"`, @@ -7837,7 +7846,7 @@ func filterSystemPromptByTemplate(template string, systemMessage string) string case "workflow-edit": removals = getWorkflowEditPromptRemovals() } - + for _, removal := range removals { if removal != "" { systemMessage = strings.ReplaceAll(systemMessage, removal, "") @@ -7860,7 +7869,7 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, aiStarttime := time.Now().UnixMilli() replacedExecution, err := GetWorkflowExecution(ctx, execution.ExecutionId) - if err == nil && len(replacedExecution.Results) > 0 && (execution.Status == "EXECUTING" || execution.Status == "WAITING") { + if err == nil && len(replacedExecution.Results) > 0 && (execution.Status == "EXECUTING" || execution.Status == "WAITING") { origStatus := execution.Status origCompleted := execution.CompletedAt origResults := execution.Results @@ -7873,11 +7882,11 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, } } - llmResponse := []byte{} - if len(aiResponseWrapper) > 0 { - if len(aiResponseWrapper[0]) > 0 { + llmResponse := []byte{} + if len(aiResponseWrapper) > 0 { + if len(aiResponseWrapper[0]) > 0 { llmResponse = aiResponseWrapper[0] - //createNextActions = false + //createNextActions = false } } @@ -7892,8 +7901,8 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, break } - if execution.Status != "EXECUTING" && execution.Status != "WAITING" { - return startNode, errors.New("Agent run already finished") + if execution.Status != "EXECUTING" && execution.Status != "WAITING" { + return startNode, errors.New("Agent run already finished") } // Metadata = org-specific context @@ -7906,25 +7915,24 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, //metadata += fmt.Sprintf("Current time: %s\n", time.Now().Format(time.RFC3339)) /* - categoryActions := GetAppCategories() - actionMetadata := "ALL Available actions sorted by category:\n" - for _, category := range categoryActions { - if category.Name == "AI" || category.Name == "Other" { - continue - } + categoryActions := GetAppCategories() + actionMetadata := "ALL Available actions sorted by category:\n" + for _, category := range categoryActions { + if category.Name == "AI" || category.Name == "Other" { + continue + } - actionMetadata += "\nCategory: " + category.Name + "\n" - for _, label := range category.ActionLabels { - actionMetadata += fmt.Sprintf("- %s\n", strings.ReplaceAll(label, "_", " ")) + actionMetadata += "\nCategory: " + category.Name + "\n" + for _, label := range category.ActionLabels { + actionMetadata += fmt.Sprintf("- %s\n", strings.ReplaceAll(label, "_", " ")) + } } - } */ if len(execution.Workflow.OrgId) == 0 && len(execution.ExecutionOrg) > 0 { execution.Workflow.OrgId = execution.ExecutionOrg } - // Validate On-Prem Configuration immediately if project.Environment != "cloud" { if os.Getenv("AI_MODEL") == "" && os.Getenv("OPENAI_MODEL") == "" { @@ -7974,7 +7982,7 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, } // Could this alleviate the need for the openai App itself? - if strings.Contains(param.Value, "$") { + if strings.Contains(param.Value, "$") { parsingBody[param.Name] = param.Value } } @@ -7982,7 +7990,7 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, // Look for conditions leading into the startNode hasConditions := false for _, branch := range execution.Workflow.Branches { - if branch.DestinationID == startNode.ID && len(branch.Conditions) > 0 { + if branch.DestinationID == startNode.ID && len(branch.Conditions) > 0 { hasConditions = true break } @@ -7990,7 +7998,7 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, llmStatusCode := 0 parsedAgentInput := "" - if hasConditions || len(parsingBody) > 0 { + if hasConditions || len(parsingBody) > 0 { marshalledBody, err := json.Marshal(parsingBody) if err == nil && len(marshalledBody) > 0 { repeaterNode := Action{} @@ -8039,7 +8047,7 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, body, err := ioutil.ReadAll(newresp.Body) if err != nil { log.Printf("[ERROR][%s] AI_AGENT_LLM_FAILURE: Failed reading response during LLM setup: %s", execution.ExecutionId, err) - } else { + } else { // Check the results of the output toolsResultMapping := SingleResult{} if err := json.Unmarshal(body, &toolsResultMapping); err != nil { @@ -8056,20 +8064,20 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, // Return IMMEDIATELY here pre-app run? branchSkipOutput := AgentOutput{ Status: "FINISHED", - StartedAt: time.Now().UnixMilli(), + StartedAt: time.Now().UnixMilli(), CompletedAt: time.Now().UnixMilli(), - ExecutionId: execution.ExecutionId, - NodeId: startNode.ID, - LLMCallCount: 0, + ExecutionId: execution.ExecutionId, + NodeId: startNode.ID, + LLMCallCount: 0, OriginalInput: originalInput, - Output: fmt.Sprintf("Branch Conditions failed: %s", reasonVal), + Output: fmt.Sprintf("Branch Conditions failed: %s", reasonVal), } marshalledOutput, err := json.Marshal(branchSkipOutput) if err != nil { - marshalledOutput = []byte(fmt.Sprintf("{\"status\":\"FINISHED\",\"output\":\"Branch Conditions failed. Failed to map reason.\",\"completed_at\":%d}", time.Now().UnixMilli())) + marshalledOutput = []byte(fmt.Sprintf("{\"status\":\"FINISHED\",\"output\":\"Branch Conditions failed. Failed to map reason.\",\"completed_at\":%d}", time.Now().UnixMilli())) } successResult := ActionResult{ @@ -8092,11 +8100,11 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, go sendAgentActionSelfRequest("SKIPPED", execution, successResult) return startNode, nil } - } + } if unmarshalErr != nil { log.Printf("[ERROR][%s] AI_AGENT_LLM_FAILURE: Failed parsing final result during LLM setup: %s", execution.ExecutionId, unmarshalErr) - } + } for paramIndex, param := range startNode.Parameters { if val, ok := mappedResult[param.Name]; ok { @@ -8123,7 +8131,7 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, if param.Name == "input" { userMessage = param.Value - + parsedAgentInput = userMessage } @@ -8131,11 +8139,11 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, enableQuestions = true } - if param.Name == "reasoning" { + if param.Name == "reasoning" { foundReasoning = strings.ToLower(strings.TrimSpace(param.Value)) } - if param.Name == "image" { + if param.Name == "image" { if strings.HasPrefix(param.Value, "http://") || strings.HasPrefix(param.Value, "https://") { imagesIncluded = append(imagesIncluded, param.Value) } else { @@ -8148,7 +8156,7 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, } } - if param.Name == "image_detail" { + if param.Name == "image_detail" { if param.Value == "low" { imageDetail = openai.ImageURLDetailLow } else if param.Value == "high" { @@ -8161,7 +8169,7 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, } if param.Name == "app_name" { - //if debug { + //if debug { // log.Printf("[DEBUG] Rewriting app_name to action") //} @@ -8207,12 +8215,12 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, requiredParams := []string{} optionalParams := []string{} for _, param := range sortedAppAction.Parameters { - if param.Name == "url" { + if param.Name == "url" { continue } - if param.Name == "body" && len(param.Example) > 0 { - if len(param.Example) > 150 { + if param.Name == "body" && len(param.Example) > 0 { + if len(param.Example) > 150 { param.Example = param.Example[:150] + "..." } @@ -8227,7 +8235,7 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, } if param.Required { - if param.Configuration && param.Name != "url" { + if param.Configuration && param.Name != "url" { continue } @@ -8268,7 +8276,7 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, sortedAppAction.Description = sortedAppAction.Description[:100] + "..." } descString = fmt.Sprintf(" # %s", sortedAppAction.Description) - } + } if descString == previousDesc { descString = "" @@ -8345,7 +8353,7 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, err := json.Unmarshal([]byte(result.Result), &mappedResult) if err != nil { log.Printf("[ERROR][%s] AI Agent (1): Failed unmarshalling result for action %s: %s", execution.ExecutionId, startNode.ID, err) - if debug { + if debug { log.Printf("[WARNING] FAILED AI AGENT THING: %s", result.Result) } break @@ -8410,7 +8418,7 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, if debug { log.Printf("[DEBUG][%s] Found existing WAITING decision at index %d (action=%s) - returning existing state", execution.ExecutionId, mappedDecision.I, mappedDecision.Action) } - + hasActiveDecision = true break } else if status == "RUNNING" { @@ -8483,7 +8491,7 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, log.Printf("[ERROR][%s] Failed to unmarshal raw response for decision at index %d: %s", execution.ExecutionId, mappedDecision.I, err) } - if parsedOutput.Status <= 0 && parsedOutput.Reason == "" { + if parsedOutput.Status <= 0 && parsedOutput.Reason == "" { } else { parsedOutput.Headers = map[string]string{} parsedOutput.Cookies = map[string]string{} @@ -8616,7 +8624,7 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, } if len(foundUserId) > 0 { - foundUser, err := GetUser(ctx, foundUserId) + foundUser, err := GetUser(ctx, foundUserId) if err == nil && len(foundUser.Id) > 0 { if len(foundUser.UserGeoInfo.Country.Name) > 0 { metadata += fmt.Sprintf("Country: %s,", foundUser.UserGeoInfo.Country.Name) @@ -8650,141 +8658,141 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, // if the user doesn't want to run anything /* - decidedApps := "" - appauth, autherr := GetAllWorkflowAppAuth(ctx, org.Id) - if autherr == nil && len(appauth) > 0 { - preferredApps := []WorkflowApp{ - WorkflowApp{ - Categories: []string{"internal"}, - Name: "shuffle datastore", - }, - } - if len(org.SecurityFramework.SIEM.Name) > 0 { - preferredApps = append(preferredApps, WorkflowApp{ - Categories: []string{"siem"}, - Name: org.SecurityFramework.SIEM.Name, - }) - } - - if len(org.SecurityFramework.EDR.Name) > 0 { - //preferredApps += strings.ToLower(org.SecurityFramework.EDR.Name) + ", " - preferredApps = append(preferredApps, WorkflowApp{ - Categories: []string{"eradication"}, - Name: org.SecurityFramework.EDR.Name, - }) - } + decidedApps := "" + appauth, autherr := GetAllWorkflowAppAuth(ctx, org.Id) + if autherr == nil && len(appauth) > 0 { + preferredApps := []WorkflowApp{ + WorkflowApp{ + Categories: []string{"internal"}, + Name: "shuffle datastore", + }, + } + if len(org.SecurityFramework.SIEM.Name) > 0 { + preferredApps = append(preferredApps, WorkflowApp{ + Categories: []string{"siem"}, + Name: org.SecurityFramework.SIEM.Name, + }) + } - if len(org.SecurityFramework.Communication.Name) > 0 { - //preferredApps += strings.ToLower(org.SecurityFramework.Cases.Name) + ", " + if len(org.SecurityFramework.EDR.Name) > 0 { + //preferredApps += strings.ToLower(org.SecurityFramework.EDR.Name) + ", " + preferredApps = append(preferredApps, WorkflowApp{ + Categories: []string{"eradication"}, + Name: org.SecurityFramework.EDR.Name, + }) + } - preferredApps = append(preferredApps, WorkflowApp{ - Categories: []string{"cases"}, - Name: org.SecurityFramework.Communication.Name, - }) - } + if len(org.SecurityFramework.Communication.Name) > 0 { + //preferredApps += strings.ToLower(org.SecurityFramework.Cases.Name) + ", " - if len(org.SecurityFramework.Cases.Name) > 0 { - //preferredApps += strings.ToLower(org.SecurityFramework.Cases.Name) + ", " + preferredApps = append(preferredApps, WorkflowApp{ + Categories: []string{"cases"}, + Name: org.SecurityFramework.Communication.Name, + }) + } - preferredApps = append(preferredApps, WorkflowApp{ - Categories: []string{"cases"}, - Name: org.SecurityFramework.Cases.Name, - }) - } + if len(org.SecurityFramework.Cases.Name) > 0 { + //preferredApps += strings.ToLower(org.SecurityFramework.Cases.Name) + ", " - if len(org.SecurityFramework.Assets.Name) > 0 { - //preferredApps += strings.ToLower(org.SecurityFramework.Assets.Name) + ", " + preferredApps = append(preferredApps, WorkflowApp{ + Categories: []string{"cases"}, + Name: org.SecurityFramework.Cases.Name, + }) + } - preferredApps = append(preferredApps, WorkflowApp{ - Categories: []string{"assets"}, - Name: org.SecurityFramework.Assets.Name, - }) - } + if len(org.SecurityFramework.Assets.Name) > 0 { + //preferredApps += strings.ToLower(org.SecurityFramework.Assets.Name) + ", " - if len(org.SecurityFramework.Network.Name) > 0 { - //preferredApps += strings.ToLower(org.SecurityFramework.Network.Name) + ", " + preferredApps = append(preferredApps, WorkflowApp{ + Categories: []string{"assets"}, + Name: org.SecurityFramework.Assets.Name, + }) + } - preferredApps = append(preferredApps, WorkflowApp{ - Categories: []string{"network"}, - Name: org.SecurityFramework.Network.Name, - }) - } + if len(org.SecurityFramework.Network.Name) > 0 { + //preferredApps += strings.ToLower(org.SecurityFramework.Network.Name) + ", " - if len(org.SecurityFramework.Intel.Name) > 0 { - //preferredApps += strings.ToLower(org.SecurityFramework.Intel.Name) + ", " + preferredApps = append(preferredApps, WorkflowApp{ + Categories: []string{"network"}, + Name: org.SecurityFramework.Network.Name, + }) + } - preferredApps = append(preferredApps, WorkflowApp{ - Categories: []string{"intel"}, - Name: org.SecurityFramework.Intel.Name, - }) - } + if len(org.SecurityFramework.Intel.Name) > 0 { + //preferredApps += strings.ToLower(org.SecurityFramework.Intel.Name) + ", " - if len(org.SecurityFramework.IAM.Name) > 0 { - //preferredApps += strings.ToLower(org.SecurityFramework.IAM.Name) + ", " - preferredApps = append(preferredApps, WorkflowApp{ - Categories: []string{"iam"}, - Name: org.SecurityFramework.IAM.Name, - }) - } + preferredApps = append(preferredApps, WorkflowApp{ + Categories: []string{"intel"}, + Name: org.SecurityFramework.Intel.Name, + }) + } - for _, auth := range appauth { - // ALWAYS append valid auth - if !auth.Validation.Valid { - continue + if len(org.SecurityFramework.IAM.Name) > 0 { + //preferredApps += strings.ToLower(org.SecurityFramework.IAM.Name) + ", " + preferredApps = append(preferredApps, WorkflowApp{ + Categories: []string{"iam"}, + Name: org.SecurityFramework.IAM.Name, + }) } - if len(auth.App.Categories) > 0 { - found := false - for _, preApp := range preferredApps { - if len(preApp.Categories) == 0 { - continue + for _, auth := range appauth { + // ALWAYS append valid auth + if !auth.Validation.Valid { + continue + } + + if len(auth.App.Categories) > 0 { + found := false + for _, preApp := range preferredApps { + if len(preApp.Categories) == 0 { + continue + } + + if ArrayContains(preApp.Categories, strings.ToLower(auth.App.Categories[0])) { + found = true + break + } } - if ArrayContains(preApp.Categories, strings.ToLower(auth.App.Categories[0])) { - found = true - break + if found { + continue } } - if found { + if len(auth.App.Categories) > 0 && strings.ToUpper(auth.App.Categories[0]) == "AI" { continue } - } - if len(auth.App.Categories) > 0 && strings.ToUpper(auth.App.Categories[0]) == "AI" { - continue + preferredApps = append(preferredApps, auth.App) } - preferredApps = append(preferredApps, auth.App) - } + // FIXME: Pre-filter before this to ensure we have good + // apps ONLY. + for _, preferredApp := range preferredApps { + if len(preferredApp.Name) == 0 { + continue + } - // FIXME: Pre-filter before this to ensure we have good - // apps ONLY. - for _, preferredApp := range preferredApps { - if len(preferredApp.Name) == 0 { - continue - } + lowername := strings.ToLower(preferredApp.Name) + if strings.Contains(decidedApps, lowername) { + continue + } - lowername := strings.ToLower(preferredApp.Name) - if strings.Contains(decidedApps, lowername) { - continue + decidedApps += lowername + ", " } - decidedApps += lowername + ", " + // Let's inject http. + if !strings.Contains(decidedApps, "http") { + decidedApps += "http, " + } } - // Let's inject http. - if !strings.Contains(decidedApps, "http") { - decidedApps += "http, " + if len(decidedApps) > 0 { + // if len(allowedActionString) == 0 { + // metadata += fmt.Sprintf("\n\nALL TOOLS: %s\n\n", decidedApps) + // } + metadata += fmt.Sprintf("\n\nALL TOOLS: %s\n\n", decidedApps) } - } - - if len(decidedApps) > 0 { - // if len(allowedActionString) == 0 { - // metadata += fmt.Sprintf("\n\nALL TOOLS: %s\n\n", decidedApps) - // } - metadata += fmt.Sprintf("\n\nALL TOOLS: %s\n\n", decidedApps) - } */ } } @@ -8797,22 +8805,22 @@ func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, //metadata += "\n" + actionMetadata } - // Due to usually NOT wanting a question back, but pure run + // Due to usually NOT wanting a question back, but pure run enableQuestionsString := ` 2. **Explicit 'Ask' Command:** - Avoid asking questions. Have an action bias and make decisions for the user! ` - if enableQuestions { + if enableQuestions { enableQuestionsString = ` 5. **Explicit 'Ask' Command:** - **Trigger:** LOWEST PRIORITY. Does the user explicitly COMMAND you to ask them for input (e.g., "Ask me for the IP")? - **Action:** Select "ask" (Category: "standalone"). - **Field "question":** The specific questions you have. Make decisions FOR the user instead of asking. Do NOT ask questions about authentication or authorization. Do NOT ask to confirm the obvious. Assume you are allowed to use the mentioned tool. Do NOT ask unless absolutely necessary. This command should generally be avoided in favor of action bias. Have as few questions as possible, but if multiple questions are required, ask one question at a time as such: "fields": [{"key": "question", "value": "question1"}, {"key": "question", "value": "question2"}]` - // FIXME: Uncomment below and add to the enableQuestionsString. New feature for auto-generating and approving new apps. The generate API docs API supports this + // FIXME: Uncomment below and add to the enableQuestionsString. New feature for auto-generating and approving new apps. The generate API docs API supports this - // If the tool is not mentioned in USER CONTEXT and you NEED them to allow those tools, set "action": "add_tool" and "tool": "EXACT toolname" and do not ask questions. If multiple tools are required, make multiple decisions - one for each required tool. Put the entire reasoning in the "reason" field - not as fields. + // If the tool is not mentioned in USER CONTEXT and you NEED them to allow those tools, set "action": "add_tool" and "tool": "EXACT toolname" and do not ask questions. If multiple tools are required, make multiple decisions - one for each required tool. Put the entire reasoning in the "reason" field - not as fields. } systemMessage += fmt.Sprintf(`### MISSION @@ -8929,7 +8937,7 @@ data_filter: agentReasoningEffort = foundReasoning } - if skipAgentWait == "true" { + if skipAgentWait == "true" { } else if len(userMessage) == 0 { log.Printf("[ERROR][%s] AI Agent: No user message/input found for action %s", execution.ExecutionId, startNode.ID) return abortAgentExecution(ctx, execution, startNode, AgentOutput{}, "no_user_message", "No user message/input found for AI Agent start") @@ -8941,7 +8949,7 @@ data_filter: initiatedBy = "system" } - if len(llmResponse) > 0 { + if len(llmResponse) > 0 { } else if !createNextActions { if strings.TrimSpace(callerName) == "" { callerName = "unknown" @@ -9004,10 +9012,10 @@ data_filter: }) } else { newMessage := openai.ChatCompletionMessage{ - Role: openai.ChatMessageRoleUser, + Role: openai.ChatMessageRoleUser, MultiContent: []openai.ChatMessagePart{ openai.ChatMessagePart{ - Type: openai.ChatMessagePartTypeText, + Type: openai.ChatMessagePartTypeText, Text: preparedContent, }, }, @@ -9015,9 +9023,9 @@ data_filter: for _, imageIncluded := range imagesIncluded { newMessage.MultiContent = append(newMessage.MultiContent, openai.ChatMessagePart{ - Type: openai.ChatMessagePartTypeImageURL, + Type: openai.ChatMessagePartTypeImageURL, ImageURL: &openai.ChatMessageImageURL{ - URL: imageIncluded, + URL: imageIncluded, Detail: imageDetail, }, }) @@ -9157,62 +9165,42 @@ data_filter: } orgStats, statsErr := GetOrgStatistics(ctx, billingOrgId) - monthlyTokensUsed := int64(0) + monthlyAppRuns := int64(0) if statsErr == nil && orgStats != nil { - monthlyTokensUsed = orgStats.MonthlyAgentTokens + convertedStats := GetCorrectedStats(orgStats) + monthlyAppRuns = convertedStats.MonthlyAppExecutions + convertedStats.MonthlyChildAppExecutions } - tokenLimit := int64(0) + appRunLimit := int64(billingOrg.SyncFeatures.AppExecutions.Limit) if project.Environment == "cloud" { - tokenLimit = int64(10_000_000) - } - if billingOrg != nil && billingOrg.SyncFeatures.AgentTokens.Active && billingOrg.SyncFeatures.AgentTokens.Limit > 0 { - tokenLimit = billingOrg.SyncFeatures.AgentTokens.Limit - } - - if tokenLimit > 0 { - estimatedCurrentTokens := EstimatePromptTokens(completionRequest.Messages) - totalTokensAfterRequest := monthlyTokensUsed + estimatedCurrentTokens - //usagePercentage := (monthlyTokensUsed * 100) / tokenLimit - - //log.Printf("[DEBUG][%s] AI_AGENT_TOKEN_USAGE: billing_org=%s exec_org=%s monthly_used=%d limit=%d usage_percent=%d%%", execution.ExecutionId, billingOrgId, execution.Workflow.OrgId, monthlyTokensUsed, tokenLimit, usagePercentage) - - if totalTokensAfterRequest > tokenLimit { - throttleKey := fmt.Sprintf("token_limit_log_%s", billingOrgId) - _, cacheErr := GetCache(ctx, throttleKey) - alreadyThrottled := cacheErr == nil - if !alreadyThrottled { - log.Printf("[ERROR][%s] AI_AGENT_TOKEN_LIMIT_EXCEEDED: billing_org=%s exec_org=%s monthly_used=%d estimated_current=%d total_would_be=%d limit=%d", execution.ExecutionId, billingOrgId, execution.Workflow.OrgId, monthlyTokensUsed, estimatedCurrentTokens, totalTokensAfterRequest, tokenLimit) - _ = SetCache(ctx, throttleKey, []byte("1"), 2*60) - go sendAITokenLimitAlert(ctx, execution, billingOrg, tokenLimit, monthlyTokensUsed) - } - return abortAgentExecution(ctx, execution, startNode, oldAgentOutput, "token_limit_exceeded", fmt.Sprintf("AI Token limit reached: %d + %d > %d. Contact support@shuffler.io to learn more, or connect to your API vendor/self-hosted model of choice to continue!", monthlyTokensUsed, estimatedCurrentTokens, tokenLimit), alreadyThrottled) + if monthlyAppRuns >= appRunLimit { + return abortAgentExecution(ctx, execution, startNode, oldAgentOutput, "app_limit_exceeded", fmt.Sprintf("AI App limit reached: %d >= %d. Contact support@shuffler.io to learn more, or connect to your API vendor/self-hosted model of choice to continue!", monthlyAppRuns, appRunLimit)) } } } bodyString := []byte{} decisionString := "" - choicesString := "" + choicesString := "" skipHttpParsing := false resultMapping := ActionResult{} openaiOutput := openai.ChatCompletionResponse{} - if agentRunLocation == "local" { + if agentRunLocation == "local" { callInfo := AiCallInfo{ - Caller: "aiAgentRunner", - OrgID: execution.Workflow.OrgId, + Caller: "aiAgentRunner", + OrgID: execution.Workflow.OrgId, } output, err := RunAiQuery( - ctx, - callInfo, - "", - "", + ctx, + callInfo, + "", + "", completionRequest, ) - if err != nil { + if err != nil { log.Printf("[ERROR][%s] AI Agent: Failed running AI query for action %s: %s", execution.ExecutionId, startNode.ID, err) return abortAgentExecution(ctx, execution, startNode, AgentOutput{}, "run_ai_query_failed", fmt.Sprintf("Failed to start AI Agent (6): %s", err.Error())) } @@ -9230,7 +9218,7 @@ data_filter: skipHttpParsing = true } else { - // FIXME: This part is almost never used anymore. Used to be necessary + // FIXME: This part is almost never used anymore. Used to be necessary // before we had the ability to run AI queries with mapping locally. // Should be removed. @@ -9267,9 +9255,7 @@ data_filter: // }, } - // Adding additional non-required params to make sure we get them parsed - - + // Adding additional non-required params to make sure we get them parsed // To ensure we get the context of an execution properly // This gives it variables to run IN CONTEXT of the current execution, @@ -9295,13 +9281,13 @@ data_filter: client.Timeout = time.Minute * 5 - // Test for whether we can ignore response wait time + // Test for whether we can ignore response wait time // This is to drastically reduce CPU use of Agent requests // 1 second = enough to read the body, which is the only major // obstacle - if skipAgentWait == "true" { - //client.Timeout = time.Second * 1 - client.Timeout = time.Millisecond * 1000 + if skipAgentWait == "true" { + //client.Timeout = time.Second * 1 + client.Timeout = time.Millisecond * 1000 fullUrl += "&skip_result_wait=true" } else { // Makes sure we wait as long as possible @@ -9332,7 +9318,7 @@ data_filter: log.Printf("[INFO][%s] Started AI Agent action %s with app '%s'. Waiting for results...", execution.ExecutionId, startNode.ID, chosenAiApp) if err != nil { - if skipAgentWait == "true" && strings.Contains(strings.ToLower(err.Error()), "timeout") { + if skipAgentWait == "true" && strings.Contains(strings.ToLower(err.Error()), "timeout") { // Question when we return here: // How do we get back to EXACTLY here when the AI is done? // Point being: we need the same data anyway. @@ -9361,7 +9347,7 @@ data_filter: log.Printf("[ERROR][%s] AI Agent: Failed reading response body from LLM: %s", execution.ExecutionId, err) return abortAgentExecution(ctx, execution, startNode, oldAgentOutput, "llm_body_read_failed", fmt.Sprintf("Failed to read LLM response body: %s", err.Error())) } - + llmStatusCode = newresp.StatusCode } @@ -9422,7 +9408,7 @@ data_filter: continue } - if debug { + if debug { log.Printf("[DEBUG][%s] AI Agent: Found body parameter which MAY contain the right user input. LEN: %d", execution.ExecutionId, len(param.Value)) } @@ -9433,7 +9419,7 @@ data_filter: } } } - } + } // Store the completion request in datastore? if len(resultMapping.Result) > 0 { @@ -9489,7 +9475,7 @@ data_filter: } // Parse the outputMap.Result to OpenAI response - // choicesString = "" + // choicesString = "" bodyMap, ok := outputMap.Body.(map[string]interface{}) if !ok { log.Printf("[ERROR][%s] AI Agent: Failed to convert body to MAP in AI Agent response. Raw response: %s", execution.ExecutionId, string(resultMapping.Result)) @@ -9514,14 +9500,14 @@ data_filter: // Edgecase handling for LLM not being available etc if len(choicesString) > 0 { - if debug { + if debug { log.Printf("[ERROR][%s] AI Agent: Found choicesString (1) in AI Agent response error handling: %s", execution.ExecutionId, choicesString) } } else if len(openaiOutput.Choices) == 0 { log.Printf("[ERROR][%s] AI Agent: No choices found in AI agent response (1). Status: %d. Raw: %s", execution.ExecutionId, outputMap.Status, bodyString) - // This is specific to OpenAI, but may work for others + // This is specific to OpenAI, but may work for others newOutput := openai.ErrorResponse{} err = json.Unmarshal(bodyString, &newOutput) if err == nil && len(newOutput.Error.Message) > 0 { @@ -9572,13 +9558,16 @@ data_filter: inputTokens := int(openaiOutput.Usage.PromptTokens) outputTokens := int(openaiOutput.Usage.CompletionTokens) totalTokens := int(openaiOutput.Usage.TotalTokens) + currentOrgId := execution.Workflow.OrgId + if len(currentOrgId) == 0 { + currentOrgId = billingOrgId + } - subOrgId := execution.Workflow.OrgId go func() { time.Sleep(time.Duration(rand.Intn(500)) * time.Millisecond) - IncrementCacheDump(ctx, billingOrgId, "agent_tokens", totalTokens) + IncrementCache(ctx, currentOrgId, "agent_tokens", totalTokens) if inputTokens > 0 { - IncrementCache(ctx, billingOrgId, "agent_input_tokens", inputTokens) + IncrementCache(ctx, currentOrgId, "agent_input_tokens", inputTokens) } if outputTokens > 0 { IncrementCache(ctx, billingOrgId, "agent_output_tokens", outputTokens) @@ -9586,25 +9575,12 @@ data_filter: if cachedTokens > 0 { IncrementCache(ctx, billingOrgId, "agent_cached_tokens", cachedTokens) } - - if billingOrgId != subOrgId { - IncrementCache(ctx, subOrgId, "agent_tokens", totalTokens) - if inputTokens > 0 { - IncrementCache(ctx, subOrgId, "agent_input_tokens", inputTokens) - } - if outputTokens > 0 { - IncrementCache(ctx, subOrgId, "agent_output_tokens", outputTokens) - } - if cachedTokens > 0 { - IncrementCache(ctx, subOrgId, "agent_cached_tokens", cachedTokens) - } - } }() - + if cachedTokens > 0 && debug { log.Printf("[DEBUG][%s] PROMPT CACHING HIT! Saved %d tokens on this request.", execution.ExecutionId, cachedTokens) } - + log.Printf("[AUDIT][%s] Incremented AI Agent usage for billing_org=%s exec_org=%s total=%d input=%d output=%d cached=%d reasoning=%d", execution.ExecutionId, billingOrgId, execution.Workflow.OrgId, totalTokens, inputTokens, outputTokens, cachedTokens, reasoningTokens) } @@ -9676,10 +9652,10 @@ data_filter: Error: errorMessage, Decisions: mappedDecisions, - ExecutionId: execution.ExecutionId, - NodeId: startNode.ID, - StartedAt: time.Now().UnixMilli(), - CompletedAt: 0, + ExecutionId: execution.ExecutionId, + NodeId: startNode.ID, + StartedAt: time.Now().UnixMilli(), + CompletedAt: 0, Memory: memorizationEngine, ExecutionMode: executionMode, @@ -9757,7 +9733,7 @@ data_filter: execution.Results[resultIndex].Result = string(agentOutputMarshalled) } - // Waiting 1 + // Waiting 1 execution.Results[resultIndex].Status = "WAITING" // Update the result in cache as actions are self-corrective @@ -9860,7 +9836,7 @@ data_filter: err = CreateOrgNotification( ctx, fmt.Sprintf("Agent - approval required for '%s'", mappedDecision.Tool), - fmt.Sprintf("Approval required during agent run."), + fmt.Sprintf("Approval required during agent run."), fmt.Sprintf("/forms/%s?authorization=%s&reference_execution=%s&source_node=%s&decision_id=%s&backend_url=%s", execution.WorkflowId, execution.Authorization, execution.ExecutionId, startNode.ID, mappedDecision.RunDetails.Id, backendUrl), execution.ExecutionOrg, false, @@ -9928,7 +9904,7 @@ data_filter: log.Printf("[DEBUG][%s] AI Agent: Decision index %d is an 'ask' action. Setting approval required to true for manual review in the UI.", execution.ExecutionId, mappedDecision.I) question := mappedDecision.Reason - if len(mappedDecision.Fields) > 0 { + if len(mappedDecision.Fields) > 0 { question = mappedDecision.Fields[0].Value } @@ -10284,7 +10260,6 @@ func GenerateSingulWorkflows(resp http.ResponseWriter, request *http.Request) { if categoryAction.ActionName == "remove" || categoryAction.ActionName == "disable" || categoryAction.ActionName == "stop" { - if workflowErr == nil && workflow.OrgId == user.ActiveOrg.Id { // Delete the workflow err = DeleteKey(ctx, "workflow", workflowId, user.ActiveOrg.Id) @@ -10293,13 +10268,13 @@ func GenerateSingulWorkflows(resp http.ResponseWriter, request *http.Request) { } /* - if debug { - log.Printf("[DEBUG] DELETING KEY: %s", deleteKey) - allWorkflows, err := GetAllWorkflowsByQuery(ctx, user, 250, "") - if err == nil { - log.Printf("\n\n[DEBUG] FOUND WORKFLOWS AFTER DELETE: %d\n\n", len(allWorkflows)) + if debug { + log.Printf("[DEBUG] DELETING KEY: %s", deleteKey) + allWorkflows, err := GetAllWorkflowsByQuery(ctx, user, 250, "") + if err == nil { + log.Printf("\n\n[DEBUG] FOUND WORKFLOWS AFTER DELETE: %d\n\n", len(allWorkflows)) + } } - } */ } else { log.Printf("[INFO] No existing workflow with ID %s to remove for category '%s'", workflowId, categoryAction.Label) @@ -10510,7 +10485,7 @@ func GenerateSingulWorkflows(resp http.ResponseWriter, request *http.Request) { if len(workflow.Actions[actionIndex].LargeImage) == 0 { - if strings.Contains(strings.ToLower(action.AppName), "agent") || strings.Contains(strings.ToLower(action.AppName), "singul") || strings.Contains(strings.ToLower(action.AppName), "integration") { + if strings.Contains(strings.ToLower(action.AppName), "agent") || strings.Contains(strings.ToLower(action.AppName), "singul") || strings.Contains(strings.ToLower(action.AppName), "integration") { workflow.Actions[actionIndex].LargeImage = "/icons/workflow-page/shuffle_agent.png" } else if debug { log.Printf("[DEBUG] Missing app image for app '%s'", action.AppName) @@ -10535,12 +10510,12 @@ func GenerateSingulWorkflows(resp http.ResponseWriter, request *http.Request) { } /* - if debug { - allWorkflows, err := GetAllWorkflowsByQuery(ctx, user, 250, "") - if err == nil { - log.Printf("\n\n[DEBUG] FOUND WORKFLOWS POST CREATE: %d\n\n", len(allWorkflows)) + if debug { + allWorkflows, err := GetAllWorkflowsByQuery(ctx, user, 250, "") + if err == nil { + log.Printf("\n\n[DEBUG] FOUND WORKFLOWS POST CREATE: %d\n\n", len(allWorkflows)) + } } - } */ resp.WriteHeader(http.StatusOK) @@ -10732,7 +10707,7 @@ func RunAiQuery(ctx context.Context, info AiCallInfo, systemMessage, userMessage } } - if len(newMessages) > 5 { + if len(newMessages) > 5 { chatCompletion.Messages = newMessages } } @@ -12941,7 +12916,7 @@ func buildMinimalWorkflow(w *Workflow) *MinimalWorkflow { } params = append(params, MinimalParameter{Name: p.Name, Value: paramValue}) } - + // Check if this action is the start node isStart := false if len(w.Start) > 0 && w.Start == a.ID { @@ -12951,7 +12926,7 @@ func buildMinimalWorkflow(w *Workflow) *MinimalWorkflow { if a.IsStartNode { isStart = true } - + minActs = append(minActs, MinimalAction{ AppName: a.AppName, AppID: a.AppID, @@ -13010,13 +12985,13 @@ func buildMinimalWorkflow(w *Workflow) *MinimalWorkflow { } params = append(params, MinimalParameter{Name: p.Name, Value: paramValue}) } - + isStart := false if len(w.Start) > 0 && w.Start == t.ID { isStart = true startTriggerID = t.ID } - + minTrigs = append(minTrigs, MinimalTrigger{ ID: t.ID, AppName: t.AppName, @@ -14447,13 +14422,13 @@ func HandleMCPMethodInitialize(request MCPRequest, user User, app WorkflowApp) ( foundServerVersion := "0.0.1" tools := MCPInitResponse{ Jsonrpc: request.Jsonrpc, - ID: request.ID, + ID: request.ID, Result: MCPToolResult{ ProtocolVersion: "2024-11-05", - Tools: []MCPTool{}, - Capabilities: MCPCapabilities{}, + Tools: []MCPTool{}, + Capabilities: MCPCapabilities{}, ServerInfo: MCPServerInfo{ - Name: "shuffle", + Name: "shuffle", Version: foundServerVersion, }, }, @@ -14461,11 +14436,11 @@ func HandleMCPMethodInitialize(request MCPRequest, user User, app WorkflowApp) ( for cnt, action := range app.Actions { tool := MCPTool{ - Name: action.Name, + Name: action.Name, Description: action.Description, InputSchema: MCPToolInputSchema{ - Type: "object", - Required: []string{}, + Type: "object", + Required: []string{}, Properties: map[string]MCPProperty{}, }, } @@ -14490,12 +14465,12 @@ func HandleMCPMethodInitialize(request MCPRequest, user User, app WorkflowApp) ( } parsedDescription := param.Description - if strings.Contains(parsedDescription, "Generated by") { + if strings.Contains(parsedDescription, "Generated by") { parsedDescription = "" } tool.InputSchema.Properties[param.Name] = MCPProperty{ - Type: "string", + Type: "string", Description: parsedDescription, } } diff --git a/blobs.go b/blobs.go index c1e243cf..578471b3 100644 --- a/blobs.go +++ b/blobs.go @@ -2597,6 +2597,14 @@ func GetBrandingAvailable(key string) bool { return false } +func GetAppRunsGrouping(key string) bool { + if key == "1e1bf9b426033f9f15e8070e007f9414d11d7fdd6a07402a9290e1a0d7965f8f" { + return true + } + + return false +} + func GetOnpremKeys() map[string]string { // key: expiry // Format: DD-MM-YYYY diff --git a/cloudSync.go b/cloudSync.go index 4d8d2d92..1cc544f2 100755 --- a/cloudSync.go +++ b/cloudSync.go @@ -918,30 +918,96 @@ func ValidateExecutionUsage(ctx context.Context, orgId string) (*Org, error) { } } - // Fix Me: Add daily stats update script to append daily stats immdediately after day change and reset monthly stats on month change - lastMonthlyReset := validationOrgStats.LastMonthlyResetMonth - currentMonth := time.Now().UTC().Month() - if int(lastMonthlyReset) != int(currentMonth) { + statsLenBefore := len(validationOrgStats.DailyStatistics) validationOrgStats = handleDailyCacheUpdate(validationOrgStats) - + if len(validationOrgStats.DailyStatistics) != statsLenBefore { err = SetOrgStatistics(ctx, *validationOrgStats, validationOrg.Id) if err != nil { - log.Printf("[ERROR] Failed setting org statistics for monthly reset for %s (%s): %s ", validationOrg.Name, validationOrg.Id, err) + log.Printf("[ERROR] Failed setting org statistics after daily rollover for %s (%s): %s ", validationOrg.Name, validationOrg.Id, err) } } totalAppExecutions := validationOrgStats.MonthlyAppExecutions + validationOrgStats.MonthlyChildAppExecutions - if validationOrg.Billing.InternalAppRunsHardLimit > 0 && totalAppExecutions > validationOrg.Billing.InternalAppRunsHardLimit { - return validationOrg, errors.New(fmt.Sprintf("Org %s (%s) has exceeded app runs hard limit (%d/%d) - Only Shuffle Support can control this metric.", validationOrg.Name, validationOrg.Id, totalAppExecutions, validationOrg.Billing.InternalAppRunsHardLimit)) + if validationOrg.SyncFeatures.AnnualAppRunsGrouping.Active == false && validationOrg.Billing.InternalAppRunsHardLimit > 0 && totalAppExecutions > validationOrg.Billing.InternalAppRunsHardLimit { + return validationOrg, errors.New(fmt.Sprintf("Org %s (%s) has exceeded app runs hard limit (%d/%d)", validationOrg.Name, validationOrg.Id, totalAppExecutions, validationOrg.Billing.InternalAppRunsHardLimit)) + } + + now := time.Now().Unix() + isExpiredAnnualPlan := false + planStartDate := int64(0) + + for _, sub := range validationOrg.Subscriptions { + if sub.Active { + subName := strings.ToLower(sub.Name) + if (strings.Contains(subName, "business") || strings.Contains(subName, "enterprise") || strings.Contains(subName, "scale")) && !strings.Contains(subName, "trial") { + planStartDate = sub.Startdate + if sub.Active && sub.Enddate > 0 && sub.Enddate < now { + isExpiredAnnualPlan = true + } + break + } + } + } + + if isExpiredAnnualPlan { + orgAdmin := User{} + for _, user := range validationOrg.Users { + if strings.ToLower(user.Role) == "admin" { + if len(user.ApiKey) > 0 && !strings.Contains(user.Username, "shuffler") { + orgAdmin = user + break + } else { + fullUser, err := GetUser(ctx, user.Id) + if err == nil && len(fullUser.ApiKey) > 0 && !strings.Contains(fullUser.Username, "shuffler") { + orgAdmin = *fullUser + break + } + } + } + } + + if len(orgAdmin.ApiKey) > 0 { + log.Printf("[AUDIT] Sending license expired request with user %s for org %s", orgAdmin.Username, validationOrg.Id) + go SendLicenseExpiredRequest(validationOrg.Id, orgAdmin.ApiKey) + } + } + + if validationOrg.SyncFeatures.AnnualAppRunsGrouping.Active == true && validationOrg.LeadInfo.Customer { + + if planStartDate > 0 { + var annualAppRuns int64 + for _, stat := range validationOrgStats.DailyStatistics { + if stat.Date.Unix() >= planStartDate { + annualAppRuns += stat.AppExecutions + stat.ChildAppExecutions + } + } + + // Set annual app runs limit as 200% of the monthly app runs limit to allow overage + annualAppRunsLimit := validationOrg.SyncFeatures.AppExecutions.Limit * 12 + if validationOrg.Billing.InternalAppRunsHardLimit > 0 && validationOrg.Billing.InternalAppRunsHardLimit <= validationOrg.SyncFeatures.AppExecutions.Limit { + annualAppRunsLimit = validationOrg.Billing.InternalAppRunsHardLimit + } else { + annualAppRunsLimit *= 2 + } + + if annualAppRuns > annualAppRunsLimit { + return validationOrg, errors.New(fmt.Sprintf("Org %s (%s) has exceeded the annual app runs limit (%d/%d)", validationOrg.Name, validationOrg.Id, annualAppRuns, annualAppRunsLimit)) + } + + return validationOrg, nil + } } // Allows partners and POV users to run workflows without limits - if validationOrg.LeadInfo.Internal || validationOrg.LeadInfo.ChannelPartner || validationOrg.LeadInfo.IntegrationPartner || validationOrg.LeadInfo.TechPartner || validationOrg.LeadInfo.DistributionPartner || validationOrg.LeadInfo.ServicePartner { + if validationOrg.LeadInfo.Internal || validationOrg.LeadInfo.ChannelPartner || validationOrg.LeadInfo.IntegrationPartner || validationOrg.LeadInfo.TechPartner || validationOrg.LeadInfo.ServicePartner { return validationOrg, nil } - // If enterprise customer or pov then don't block them - if (validationOrg.LeadInfo.Customer || validationOrg.LeadInfo.POV) && validationOrg.SyncFeatures.AppExecutions.Limit >= 300000 { + if validationOrg.LeadInfo.Customer && validationOrg.SyncFeatures.AppExecutions.Limit >= 300000 { + extendedLimit := validationOrg.SyncFeatures.AppExecutions.Limit * 10 + if totalAppExecutions >= extendedLimit { + return validationOrg, errors.New(fmt.Sprintf("Org %s (%s) has exceeded the monthly app executions limit (%d/%d)", validationOrg.Name, validationOrg.Id, totalAppExecutions, extendedLimit)) + } return validationOrg, nil } diff --git a/db-connector.go b/db-connector.go index a958be88..21428ad1 100755 --- a/db-connector.go +++ b/db-connector.go @@ -131,7 +131,7 @@ func SetOrgStatistics(ctx context.Context, stats ExecutionInfo, id string) error } stat.Date = stat.Date.UTC() - statdate := stat.Date.Format("2006-12-30") + statdate := stat.Date.Format("2006-01-02") if !ArrayContains(allDates, statdate) { newDaily = append(newDaily, stat) allDates = append(allDates, statdate) @@ -1537,6 +1537,86 @@ func IncrementCacheDump(ctx context.Context, orgId, dataType string, amount ...i } } + if len(tmpOrgDetail.ManagerOrgs) > 0 && (dataType == "agent_tokens") { + for _, managerOrg := range tmpOrgDetail.ManagerOrgs { + if len(managerOrg.Id) == 36 { + IncrementCache(ctx, managerOrg.Id, "childorg_agent_tokens", int(dbDumpInterval)) + } + } + } + + if len(tmpOrgDetail.ManagerOrgs) > 0 && (dataType == "agent_input_tokens") { + for _, managerOrg := range tmpOrgDetail.ManagerOrgs { + if len(managerOrg.Id) == 36 { + IncrementCache(ctx, managerOrg.Id, "childorg_agent_input_tokens", int(dbDumpInterval)) + } + } + } + + if len(tmpOrgDetail.ManagerOrgs) > 0 && (dataType == "agent_output_tokens") { + for _, managerOrg := range tmpOrgDetail.ManagerOrgs { + if len(managerOrg.Id) == 36 { + IncrementCache(ctx, managerOrg.Id, "childorg_agent_output_tokens", int(dbDumpInterval)) + } + } + } + + if len(tmpOrgDetail.ManagerOrgs) > 0 && (dataType == "send_sms") { + for _, managerOrg := range tmpOrgDetail.ManagerOrgs { + if len(managerOrg.Id) == 36 { + IncrementCache(ctx, managerOrg.Id, "childorg_send_sms", int(dbDumpInterval)) + } + } + } + + if len(tmpOrgDetail.ManagerOrgs) > 0 && (dataType == "send_mail") { + for _, managerOrg := range tmpOrgDetail.ManagerOrgs { + if len(managerOrg.Id) == 36 { + IncrementCache(ctx, managerOrg.Id, "childorg_send_mail", int(dbDumpInterval)) + } + } + } + + if len(tmpOrgDetail.ManagerOrgs) > 0 && (dataType == "agent_cached_tokens") { + for _, managerOrg := range tmpOrgDetail.ManagerOrgs { + if len(managerOrg.Id) == 36 { + IncrementCache(ctx, managerOrg.Id, "childorg_agent_cached_tokens", int(dbDumpInterval)) + } + } + } + + if len(tmpOrgDetail.ManagerOrgs) > 0 && (dataType == "agent_executions") { + for _, managerOrg := range tmpOrgDetail.ManagerOrgs { + if len(managerOrg.Id) == 36 { + IncrementCache(ctx, managerOrg.Id, "child_org_agent_executions", int(dbDumpInterval)) + } + } + } + + if len(tmpOrgDetail.ManagerOrgs) > 0 && (dataType == "agent_executions_successful") { + for _, managerOrg := range tmpOrgDetail.ManagerOrgs { + if len(managerOrg.Id) == 36 { + IncrementCache(ctx, managerOrg.Id, "child_org_agent_executions_successful", int(dbDumpInterval)) + } + } + } + + if len(tmpOrgDetail.ManagerOrgs) > 0 && (dataType == "agent_executions_failed") { + for _, managerOrg := range tmpOrgDetail.ManagerOrgs { + if len(managerOrg.Id) == 36 { + IncrementCache(ctx, managerOrg.Id, "child_org_agent_executions_failed", int(dbDumpInterval)) + } + } + } + + if len(tmpOrgDetail.ManagerOrgs) > 0 && (dataType == "agent_max_loops_hit") { + for _, managerOrg := range tmpOrgDetail.ManagerOrgs { + if len(managerOrg.Id) == 36 { + IncrementCache(ctx, managerOrg.Id, "child_org_agent_max_loops_hit", int(dbDumpInterval)) + } + } + } + concurrentTxn := false errMsg := "" @@ -2155,11 +2235,11 @@ func getExecutionFileValue(ctx context.Context, workflowExecution WorkflowExecut obj := bucket.Object(fullParsedPath) fileReader, err := obj.NewReader(ctx) if err != nil { - if debug { + if debug { log.Printf("[DEBUG] Failed reading file '%s' from bucket %s: %s. Will try with alternative solution.", fullParsedPath, bucketName, err) } - // Cache sip for the minute + // Cache sip for the minute SetCache(ctx, cacheKey, []byte{}, 1) if projectName != "shuffler" { @@ -2169,7 +2249,7 @@ func getExecutionFileValue(ctx context.Context, workflowExecution WorkflowExecut fileReader, err = obj.NewReader(ctx) if err != nil { //log.Printf("[ERROR] Failed reading file '%s' again from bucket %s: %s", fullParsedPath, bucketName, err) - + return "", err } } else { @@ -2312,7 +2392,7 @@ func Fixexecution(ctx context.Context, workflowExecution WorkflowExecution) (Wor workflowExecution.Workflow.Validation = validation } - //if debug { + //if debug { // log.Printf("\n\n[DEBUG][%s] EXEC CHECK? Actions: %d, Results: %d\n\n", workflowExecution.ExecutionId, len(workflowExecution.Workflow.Actions), len(workflowExecution.Results)) //} @@ -2328,7 +2408,7 @@ func Fixexecution(ctx context.Context, workflowExecution WorkflowExecution) (Wor // Very weird edgecase handling for agent cleanup // This is for auto-correctiveness of executions - if len(workflowExecution.Workflow.Actions) == 1 && action.Name == "agent" && innerresult.Action.Name == "agent" && innerresult.Action.ID == "" { + if len(workflowExecution.Workflow.Actions) == 1 && action.Name == "agent" && innerresult.Action.Name == "agent" && innerresult.Action.ID == "" { innerresult.Action.ID = action.ID innerresult.Action.AppName = "AI Agent" } @@ -2396,25 +2476,25 @@ func Fixexecution(ctx context.Context, workflowExecution WorkflowExecution) (Wor } finishFound := false - for decisionIndex, decision := range mappedOutput.Decisions { - if decision.Action == "finish" || decision.Category == "finish" { + for decisionIndex, decision := range mappedOutput.Decisions { + if decision.Action == "finish" || decision.Category == "finish" { - if decision.RunDetails.Status != "FINISHED" { - if mappedOutput.Decisions[decisionIndex].RunDetails.StartedAt == 0 { - mappedOutput.Decisions[decisionIndex].RunDetails.StartedAt = time.Now().UnixMilli() + if decision.RunDetails.Status != "FINISHED" { + if mappedOutput.Decisions[decisionIndex].RunDetails.StartedAt == 0 { + mappedOutput.Decisions[decisionIndex].RunDetails.StartedAt = time.Now().UnixMilli() } - mappedOutput.Decisions[decisionIndex].RunDetails.CompletedAt = time.Now().UnixMilli() + mappedOutput.Decisions[decisionIndex].RunDetails.CompletedAt = time.Now().UnixMilli() mappedOutput.Decisions[decisionIndex].RunDetails.Status = "FINISHED" decisionsUpdated = true } - + finishFound = true } } - if finishFound { - //if debug { + if finishFound { + //if debug { // log.Printf("[DEBUG][%s] SELF AGENT FINISH FOUND", workflowExecution.ExecutionId) //} @@ -2450,7 +2530,7 @@ func Fixexecution(ctx context.Context, workflowExecution WorkflowExecution) (Wor if decision.Action == "finish" { finishDecisionFound = true - if decision.RunDetails.Status == "" { + if decision.RunDetails.Status == "" { decision.RunDetails.Status = "FINISHED" mappedOutput.Decisions[decisionIndex].RunDetails.Status = "FINISHED" } @@ -2655,7 +2735,7 @@ func Fixexecution(ctx context.Context, workflowExecution WorkflowExecution) (Wor }() } } else if (result.Status == "" || result.Status == "WAITING") && mappedOutput.Status == "FINISHED" { - if debug { + if debug { log.Printf("[INFO][%s] Agent action %s marked as FINISHED, updating result status to SUCCESS.", workflowExecution.ExecutionId, action.ID) } @@ -4882,7 +4962,7 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User, maxAmount int, curso _, err = it.Next(&innerWorkflow) if err != nil { if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") { - if debug { + if debug { //log.Printf("[DEBUG] Workflow load iterator issue: %s", err) } @@ -4896,7 +4976,7 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User, maxAmount int, curso } if innerWorkflow.Public { - //if debug { + //if debug { // log.Printf("[DEBUG] Skipping public workflow %s (%s) for org %s", innerWorkflow.Name, innerWorkflow.ID, user.ActiveOrg.Id) //} @@ -4904,7 +4984,7 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User, maxAmount int, curso } if innerWorkflow.Hidden { - //if debug { + //if debug { // log.Printf("[DEBUG] Skipping HIDDEN workflow %s (%s) for org %s", innerWorkflow.Name, innerWorkflow.ID, user.ActiveOrg.Id) //} @@ -4928,12 +5008,12 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User, maxAmount int, curso } } - // Fallback for when the iterator fails due to a datastore issue + // Fallback for when the iterator fails due to a datastore issue // (e.g. "cannot load field" error) and similar if err != iterator.Done { log.Printf("[WARNING] Failed fetching workflow results for org %s: %v", user.ActiveOrg.Id, err) - // Check if query contains edited or not + // Check if query contains edited or not if strings.Contains(fmt.Sprintf("%s", err), "FailedPrecondition desc") && strings.Contains(fmt.Sprintf("%s", query), "edited") { log.Printf("[ERROR] Retrying workflow query without Edited sort due to error: %s", err) @@ -4969,7 +5049,7 @@ func GetAllWorkflowsByQuery(ctx context.Context, user User, maxAmount int, curso }) if len(workflows) > maxAmount { - if debug { + if debug { log.Printf("[WARNING] Found %d workflows for user %s (%s) in org %s, but limiting to %d", len(workflows), user.Username, user.Id, user.ActiveOrg.Id, maxAmount) } @@ -5107,6 +5187,60 @@ func GetOrgByCreatorId(ctx context.Context, id string) (*Org, error) { return curOrg, nil } +var defaultAlertThresholdPercentages = []int{50, 75, 90, 100} + +func isOnpremAlertEligible(org *Org) bool { + if !org.CloudSyncActive { + return false + } + + return org.LeadInfo.EnterpriseLicenseOnprem || + org.LeadInfo.BusinessLicenseOnprem || + org.LeadInfo.ScaleLicenseOnpremCustomer +} + +func mergeDefaultAlertThresholds(thresholds []AlertThreshold, limit int64) []AlertThreshold { + existingPercentages := map[int]bool{} + for _, threshold := range thresholds { + existingPercentages[threshold.Percentage] = true + } + + for _, percentage := range defaultAlertThresholdPercentages { + if !existingPercentages[percentage] { + thresholds = append(thresholds, AlertThreshold{ + Percentage: percentage, + Count: int(float64(percentage) / 100 * float64(limit)), + }) + } + } + + return thresholds +} + +func addDefaultAlertThresholds(org *Org) bool { + changed := false + + if !org.Billing.DefaultAlertsApplied { + limit := org.SyncFeatures.AppExecutions.Limit + if limit > 0 { + org.Billing.AlertThreshold = mergeDefaultAlertThresholds(org.Billing.AlertThreshold, limit) + org.Billing.DefaultAlertsApplied = true + changed = true + } + } + + if !org.Billing.DefaultOnpremAlertsApplied { + onpremLimit := org.SyncFeatures.OnpremAppExecutions.Limit + if onpremLimit > 0 && isOnpremAlertEligible(org) { + org.Billing.OnpremAlertThreshold = mergeDefaultAlertThresholds(org.Billing.OnpremAlertThreshold, onpremLimit) + org.Billing.DefaultOnpremAlertsApplied = true + changed = true + } + } + + return changed +} + // ListBooks returns a list of books, ordered by title. // Handles org grabbing and user / org migrations func GetOrg(ctx context.Context, id string) (*Org, error) { @@ -5143,6 +5277,14 @@ func GetOrg(ctx context.Context, id string) (*Org, error) { if curOrg.Id == "" { return curOrg, errors.New("Org doesn't exist") } else { + billingBackup := curOrg.Billing + if addDefaultAlertThresholds(curOrg) { + err := SetOrg(ctx, *curOrg, curOrg.Id) + if err != nil { + log.Printf("[ERROR] Failed persisting default alert thresholds for org %s: %s", curOrg.Id, err) + curOrg.Billing = billingBackup + } + } return curOrg, nil } } @@ -5295,6 +5437,14 @@ func GetOrg(ctx context.Context, id string) (*Org, error) { } curOrg.Priorities = newPriorities + billingBackup := curOrg.Billing + if addDefaultAlertThresholds(curOrg) { + err := SetOrg(ctx, *curOrg, curOrg.Id) + if err != nil { + log.Printf("[ERROR] Failed persisting default alert thresholds for org %s: %s", curOrg.Id, err) + curOrg.Billing = billingBackup + } + } if project.CacheDb { neworg, err := json.Marshal(curOrg) if err != nil { @@ -9340,11 +9490,16 @@ func GetWorkflowQueue(ctx context.Context, id string, limit int, inputEnv ...Env if project.Environment != "cloud" && len(inputEnv) > 0 && len(executions) > 0 { env := inputEnv[0] - orgId := env.OrgId - org, err := GetOrg(ctx, orgId) + + org, err := GetFirstOrg(ctx) if err != nil { - log.Printf("[ERROR] Failed getting org %s for queue: %s", orgId, err) + log.Printf("[ERROR] Failed getting parent org directly for queue: %s", err) + return ExecutionRequestWrapper{ + Data: executions, + }, nil + } + if len(org.Id) == 0 { return ExecutionRequestWrapper{ Data: executions, }, nil @@ -9375,15 +9530,78 @@ func GetWorkflowQueue(ctx context.Context, id string, limit int, inputEnv ...Env license := checkNoInternet() if license.Valid { - limit = limit * 2 + if license.AppRunsGrouping { + limit = limit * 12 + if licenseOrg.Billing.InternalAppRunsHardLimit > 0 && licenseOrg.Billing.InternalAppRunsHardLimit <= licenseOrg.SyncFeatures.AppExecutions.Limit { + limit = licenseOrg.Billing.InternalAppRunsHardLimit + } + + var planStartDate int64 + for _, sub := range licenseOrg.Subscriptions { + if sub.Active { + subName := strings.ToLower(sub.Name) + if strings.Contains(subName, "business") || strings.Contains(subName, "enterprise") { + planStartDate = sub.Startdate + break + } + } + } + + var annualAppRuns int64 + if planStartDate > 0 { + for _, stat := range stats.DailyStatistics { + if stat.Date.Unix() >= planStartDate { + annualAppRuns += stat.AppExecutions + stat.ChildAppExecutions + } + } + } + totalAppExecutions = annualAppRuns + } else { + limit = limit * 2 + } + + } else if licenseOrg.CloudSync && licenseOrg.SyncFeatures.AnnualAppRunsGrouping.Active { + limit = limit * 12 * 2 + + if licenseOrg.Billing.InternalAppRunsHardLimit > 0 && licenseOrg.Billing.InternalAppRunsHardLimit <= licenseOrg.SyncFeatures.AppExecutions.Limit { + limit = licenseOrg.Billing.InternalAppRunsHardLimit + } + + var planStartDate int64 + for _, sub := range licenseOrg.Subscriptions { + if sub.Active { + subName := strings.ToLower(sub.Name) + if strings.Contains(subName, "business") || strings.Contains(subName, "enterprise") || strings.Contains(subName, "scale") { + planStartDate = sub.Startdate + break + } + } + } + + var annualAppRuns int64 + if planStartDate > 0 { + for _, stat := range stats.DailyStatistics { + if stat.Date.Unix() >= planStartDate { + annualAppRuns += stat.AppExecutions + stat.ChildAppExecutions + } + } + } + totalAppExecutions = annualAppRuns + } else if licenseOrg.CloudSync && licenseOrg.SyncFeatures.AppExecutions.Limit >= 300000 { + limit = limit * 10 + + if licenseOrg.Billing.InternalAppRunsHardLimit > 0 { + limit = licenseOrg.Billing.InternalAppRunsHardLimit + } + } - shouldSkipRateLimit := false - if licenseOrg.CloudSync && !license.Valid && licenseOrg.SyncFeatures.AppExecutions.Limit >= 300000 { - shouldSkipRateLimit = true + if debug { + log.Printf("[INFO] total app executions in the queue is: %v", totalAppExecutions) + log.Printf("[INFO] app runs limit in the queue is: %v", limit) } - if !shouldSkipRateLimit && totalAppExecutions > limit { + if totalAppExecutions > limit { cacheKey := fmt.Sprintf("org-%s-last-queue-send", orgId) currentTime := time.Now().Unix() lastSendCache, err := GetCache(ctx, cacheKey) @@ -9413,7 +9631,7 @@ func GetWorkflowQueue(ctx context.Context, id string, limit int, inputEnv ...Env } else { if len(executions) > 1 { - log.Printf("[INFO] Rate limiting (3): Org %s exceeded the 25K app run quota for non-licensed users (current queued: %d, current month usage: %d). To increase scale, upgrade to an Enterprise license.", orgId, len(executions), totalAppExecutions) + log.Printf("[INFO] Rate limiting (3): Org %s exceeded the %v app run montly quota (current queued: %d, current month usage: %d). To increase scale, upgrade to an Enterprise license.", orgId, limit, len(executions), totalAppExecutions) executions = executions[0:1] } @@ -11356,7 +11574,7 @@ func GetApikey(ctx context.Context, apikey string) (User, error) { } if debug { - log.Printf("[DEBUG] API key cache miss; looking up user") + log.Printf("[DEBUG] API key cache miss; looking up user") } if project.DbType == "opensearch" { @@ -12092,7 +12310,7 @@ func GetOrgNotifications(ctx context.Context, orgId string) ([]Notification, err "size": 1000, "sort": map[string]interface{}{ "updated_at": map[string]interface{}{ - "order": "desc", + "order": "desc", "unmapped_type": "long", }, }, @@ -15045,7 +15263,7 @@ func SetDatastoreKeyBulk(ctx context.Context, allKeys []CacheKeyData) ([]Datasto oldDoc := config.Value newDoc := cacheData.Value - if debug { + if debug { log.Printf("\n\nOLD: %s\n\nNEW: %s\n\n", oldDoc, newDoc) } @@ -15070,9 +15288,9 @@ func SetDatastoreKeyBulk(ctx context.Context, allKeys []CacheKeyData) ([]Datasto break } - // This NEVER triggers. RLS just returns the merged JSON - // and we trust it. If we don't trust it, we can set - // ruleValid to false above. + // This NEVER triggers. RLS just returns the merged JSON + // and we trust it. If we don't trust it, we can set + // ruleValid to false above. if !ruleValid { // Break out if debug { @@ -15081,8 +15299,8 @@ func SetDatastoreKeyBulk(ctx context.Context, allKeys []CacheKeyData) ([]Datasto keyUpdated = false - cacheData.Existed = true - cacheData.Changed = keyUpdated + cacheData.Existed = true + cacheData.Changed = keyUpdated datastoreKeys <- *datastore.NameKey(nameKey, datastoreId, nil) cacheKeys <- cacheData return @@ -16523,9 +16741,12 @@ func checkNoInternet() OnpremLicense { Active: false, Limit: 25000, }, - Timeout: "", - Branding: false, + Timeout: "", + Branding: false, + StartDate: "", + AppRunsGrouping: false, } + licenseKey := os.Getenv("SHUFFLE_LICENSE") if len(licenseKey) == 0 { return license @@ -16581,6 +16802,24 @@ func checkNoInternet() OnpremLicense { brandingHash := sha256.Sum256([]byte(branding)) encodedBranding := hex.EncodeToString(brandingHash[:]) + + startDate := "" + if len(licenseParts) > 5 { + startDate = licenseParts[5] + } + + startDateHash := sha256.Sum256([]byte(startDate)) + encodedStartDate := hex.EncodeToString(startDateHash[:]) + + // check if annual appruns grouping available + appRunsGrouping := "" + if len(licenseParts) > 6 { + appRunsGrouping = licenseParts[6] + } + + appRunsGroupingHash := sha256.Sum256([]byte(appRunsGrouping)) + encodedAppRunsGrouping := hex.EncodeToString(appRunsGroupingHash[:]) + // Returns a map[sha256]timeout string onpremKeys := GetOnpremKeys() if timeout, ok := onpremKeys[encodedString]; ok { @@ -16641,6 +16880,19 @@ func checkNoInternet() OnpremLicense { } } + if len(startDate) > 0 && len(encodedStartDate) > 0 { + if startDate, ok := onpremKeys[encodedStartDate]; ok { + license.StartDate = startDate + } + } + + if len(appRunsGrouping) > 0 && len(encodedAppRunsGrouping) > 0 { + appRuns := GetAppRunsGrouping(encodedAppRunsGrouping) + license.AppRunsGrouping = appRuns + } else { + license.AppRunsGrouping = false + } + return license } else { log.Printf("[ERROR] License key has expired on %s", timeout) @@ -17534,37 +17786,37 @@ func GetAllCacheKeys(ctx context.Context, orgId string, category string, max int if parentOrgDepth >= 3 { log.Printf("[ERROR] Reached maximum parent org lookup depth (%d) for org %s. Skipping parent org cache lookup to prevent infinite recursion.", parentOrgDepth, orgId) } else { - parentOrg, err := GetOrg(ctx, foundOrg.CreatorOrg) - if err != nil { + parentOrg, err := GetOrg(ctx, foundOrg.CreatorOrg) + if err != nil { if debug { log.Printf("[DEBUG] Could not find parent org %s for org %s (possibly in different region): %s", foundOrg.CreatorOrg, orgId, err) } - } else { + } else { parentOrgCache, _, err := GetAllCacheKeys(ctx, parentOrg.Id, "", max, inputcursor, cleanupDepth, parentOrgDepth+1) - if err != nil { + if err != nil { if debug { log.Printf("[DEBUG] Failed getting parent org cache keys for org %s: %s", parentOrg.Id, err) } - } else { - if debug { - //log.Printf("[DEBUG] Loaded %d parent org cache keys for org %s. Validating if child org %s should get the keys", len(parentOrgCache), parentOrg.Id, orgId) - } + } else { + if debug { + //log.Printf("[DEBUG] Loaded %d parent org cache keys for org %s. Validating if child org %s should get the keys", len(parentOrgCache), parentOrg.Id, orgId) + } - for _, parentCache := range parentOrgCache { - /* - if debug && len(parentCache.SuborgDistribution) > 0 { - log.Printf("[DEBUG] Parent org %s keys: %#v", parentOrg.Id, parentCache.SuborgDistribution) - } - */ + for _, parentCache := range parentOrgCache { + /* + if debug && len(parentCache.SuborgDistribution) > 0 { + log.Printf("[DEBUG] Parent org %s keys: %#v", parentOrg.Id, parentCache.SuborgDistribution) + } + */ - if !ArrayContains(parentCache.SuborgDistribution, orgId) { - continue - } + if !ArrayContains(parentCache.SuborgDistribution, orgId) { + continue + } - // Clean up just in case - parentCache.PublicAuthorization = "" - parentCache.SuborgDistribution = []string{orgId} - cacheKeys = append(cacheKeys, parentCache) + // Clean up just in case + parentCache.PublicAuthorization = "" + parentCache.SuborgDistribution = []string{orgId} + cacheKeys = append(cacheKeys, parentCache) } } } diff --git a/shared.go b/shared.go index d8211f52..f1881dbb 100644 --- a/shared.go +++ b/shared.go @@ -1479,6 +1479,13 @@ func HandleGetOrg(resp http.ResponseWriter, request *http.Request) { org.LeadInfo = LeadInfo{} } + // Sort subscriptions: active first, inactive last + sort.Slice(org.Subscriptions, func(i, j int) bool { + return org.Subscriptions[i].Active && !org.Subscriptions[j].Active + }) + + org.CloudSync = org.CloudSyncActive + newjson, err := json.Marshal(org) if err != nil { log.Printf("[ERROR] Failed unmarshal of org %s (%s): %s", org.Name, org.Id, err) @@ -2925,12 +2932,12 @@ func HandleSetEnvironments(resp http.ResponseWriter, request *http.Request) { } if project.Environment == "cloud" { - //foundOrg, err := GetOrg(ctx, user.ActiveOrg.Id) - //if err != nil { - // resp.WriteHeader(401) - // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed find your organization"}`))) - // return - //} + foundOrg, err := GetOrg(ctx, user.ActiveOrg.Id) + if err != nil { + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed find your organization"}`))) + return + } // FIXME: Removed need for syncfeatures to be enabled // September 2022 @@ -2941,6 +2948,28 @@ func HandleSetEnvironments(resp http.ResponseWriter, request *http.Request) { // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Adding multiple environments requires an active hybrid, enterprise or MSSP subscription"}`))) // return //} + + if len(foundOrg.CreatorOrg) > 0 { + foundOrg, err = GetOrg(ctx, foundOrg.CreatorOrg) + if err != nil { + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed find your organization"}`))) + return + } + } + + envs, err := GetEnvironments(ctx, foundOrg.Id) + if err != nil { + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to get environments of organization"}`))) + return + } + + if int64(len(envs)) > foundOrg.SyncFeatures.MultiEnv.Limit { + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "You have reached the limit of %d environments for your subscription. Upgrade to an enterprise plan or contact support@shuffler.io for more info."}`, foundOrg.SyncFeatures.MultiEnv.Limit))) + return + } } if project.Environment == "onprem" { @@ -13582,7 +13611,7 @@ func BuildBaseSubscription(org Org, monthlyExecLimit int64) PaymentSubscription if project.Environment == "cloud" { // Cloud licenses if monthlyExecLimit >= 300000 { - planName = "Cloud Enterprise License" + planName = "Business License (Cloud)" supportLevel = "Enterprise Support" features = []string{ "∞ Days Workflow Backup", @@ -13598,7 +13627,7 @@ func BuildBaseSubscription(org Org, monthlyExecLimit int64) PaymentSubscription } amount = "870" // Just for placeholder } else if monthlyExecLimit >= 12000 { - planName = "Cloud Scale License" + planName = "Scale License (Cloud)" supportLevel = "Standard Support" features = []string{ "30 Days workflow run history", @@ -13608,7 +13637,7 @@ func BuildBaseSubscription(org Org, monthlyExecLimit int64) PaymentSubscription } amount = fmt.Sprintf("%d", int64(((monthlyExecLimit-2000)/10000)*32)) // Calculate based on app runs: (paid_runs / 10k) * $32 } else if monthlyExecLimit >= 2000 && monthlyExecLimit < 12000 { - planName = "Free License" + planName = "Scale License (Cloud Trial)" supportLevel = "Community Support" features = []string{ "All 2500+ Apps", @@ -13805,6 +13834,71 @@ func HandleEditOrg(resp http.ResponseWriter, request *http.Request) { return } + if tmpData.Editing == "license_expired" { + // 1. Check if editing org is on mark as customer and opensource if yes than only countinue this checks + if org.LeadInfo.Customer { + // 2. Check if editing org have subscription active and it is ended if yes than only continue + now := time.Now().Unix() + hasEndedSubscription := false + for _, sub := range org.Subscriptions { + subName := strings.ToLower(sub.Name) + if (strings.Contains(subName, "enterprise") || strings.Contains(subName, "business")) && sub.Active && sub.Enddate > 0 && sub.Enddate < now { + hasEndedSubscription = true + break + } + } + + if hasEndedSubscription { + // 3. If both of the above conditions are met than it's orgs onpremappruns limit as 25K, set subscription active as false, tenants limit 3, environmennt limit as 1 and branding as false + org.SyncFeatures.OnpremAppExecutions.Limit = 25000 + org.SyncFeatures.AppExecutions.Limit = 2000 + org.SyncFeatures.AnnualAppRunsGrouping.Active = false + var newSubs []PaymentSubscription + hasBaseSubscription := false + for i := range org.Subscriptions { + subName := strings.ToLower(org.Subscriptions[i].Name) + if strings.Contains(subName, "enterprise") || strings.Contains(subName, "business") { + org.Subscriptions[i].Active = false + } + if strings.Contains(subName, "free") || strings.Contains(subName, "open source") { + hasBaseSubscription = true + } + newSubs = append(newSubs, org.Subscriptions[i]) + } + + if !hasBaseSubscription { + newSubs = append(newSubs, BuildBaseSubscription(*org, 2000)) + } + org.Subscriptions = newSubs + org.SyncFeatures.MultiTenant.Limit = 3 + org.SyncFeatures.MultiEnv.Limit = 1 + org.SyncFeatures.Branding.Active = false + + org.LeadInfo.Customer = false + org.LeadInfo.OpenSource = false + org.LeadInfo.BusinessLicenseOnprem = false + org.LeadInfo.BusinessLicenseCloud = false + org.LeadInfo.EnterpriseLicenseCloud = false + org.LeadInfo.EnterpriseLicenseOnprem = false + org.LeadInfo.ShuffleEnterpriseLicenseOldCustomer = false + // 4. Update above information in org and return sucess true and don't continue this furthus + err = SetOrg(ctx, *org, org.Id) + if err != nil { + log.Printf("[ERROR] Failed to update org %s on license expiry: %v", org.Id, err) + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false}`)) + return + } + resp.WriteHeader(200) + resp.Write([]byte(`{"success": true}`)) + return + } + } + resp.WriteHeader(200) + resp.Write([]byte(`{"success": true}`)) + return + } + // Allow editing a specific subscription card from UI except Eula and Reference if tmpData.Editing == "subscription_update" { // Find subscription by ID (SubscriptionIndex now holds the ID string) @@ -13883,6 +13977,34 @@ func HandleEditOrg(resp http.ResponseWriter, request *http.Request) { return } + if tmpData.Editing == "subscription_delete" && !user.SupportAccess { + resp.WriteHeader(403) + resp.Write([]byte(`{"success": false, "reason": "Support access required"}`)) + return + } + + if tmpData.Editing == "subscription_delete" { + var filtered []PaymentSubscription + for _, sub := range org.Subscriptions { + if sub.Id != tmpData.SubscriptionIndex { + filtered = append(filtered, sub) + } + } + org.Subscriptions = filtered + + if err := SetOrg(ctx, *org, org.Id); err != nil { + log.Printf("[WARNING] Failed to delete subscription for org %s: %s", org.Id, err) + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false}`)) + return + } + + log.Printf("[AUDIT] Support user %s deleted subscription %s from org %s", user.Username, tmpData.SubscriptionIndex, org.Id) + resp.WriteHeader(200) + resp.Write([]byte(`{"success": true}`)) + return + } + sendOrgUpdaterHook := false if len(tmpData.Image) > 0 { org.Image = tmpData.Image @@ -13942,7 +14064,7 @@ func HandleEditOrg(resp http.ResponseWriter, request *http.Request) { } */ - // Update Billing email alert threshold + // Update Billing email alert threshold (cloud) tmpDataAlert := tmpData.Billing.AlertThreshold orgAlertThreshold := org.Billing.AlertThreshold @@ -13958,12 +14080,64 @@ func HandleEditOrg(resp http.ResponseWriter, request *http.Request) { } } } + + // Update Billing email alert threshold (onprem) - independent list from cloud above + tmpDataOnpremAlert := tmpData.Billing.OnpremAlertThreshold + orgOnpremAlertThreshold := org.Billing.OnpremAlertThreshold + + if len(tmpDataOnpremAlert) > 0 { + if len(tmpDataOnpremAlert) != len(orgOnpremAlertThreshold) { + org.Billing.OnpremAlertThreshold = tmpData.Billing.OnpremAlertThreshold + } else { + for i := 0; i < len(tmpDataOnpremAlert); i++ { + if tmpDataOnpremAlert[i].Percentage != orgOnpremAlertThreshold[i].Percentage || tmpDataOnpremAlert[i].Count != orgOnpremAlertThreshold[i].Count { + org.Billing.OnpremAlertThreshold = tmpData.Billing.OnpremAlertThreshold + break + } + } + } + } if tmpData.Editing == "app_runs_hard_limit" && tmpData.Billing.AppRunsHardLimit != org.Billing.AppRunsHardLimit { org.Billing.AppRunsHardLimit = tmpData.Billing.AppRunsHardLimit } - if user.SupportAccess && tmpData.Editing == "internal_appruns_hard_limit" && tmpData.Billing.InternalAppRunsHardLimit != org.Billing.InternalAppRunsHardLimit { + if tmpData.Editing == "app_runs_grouping" && !org.SyncFeatures.AnnualAppRunsGrouping.Active && !user.SupportAccess { + org.SyncFeatures.AnnualAppRunsGrouping.Active = tmpData.SyncFeatures.AnnualAppRunsGrouping.Active + } + + if tmpData.Editing == "internal_appruns_hard_limit" && tmpData.Billing.InternalAppRunsHardLimit != org.Billing.InternalAppRunsHardLimit { + if !user.SupportAccess { + if org.SyncFeatures.AnnualAppRunsGrouping.Active { + // Allow 200% app runs hard limit for annual plan + maxAllowed := org.SyncFeatures.AppExecutions.Limit * 12 * 2 + if tmpData.Billing.InternalAppRunsHardLimit > maxAllowed { + resp.WriteHeader(400) + resp.Write([]byte(`{"success": false, "reason": "Hard limit cannot exceed 200% of the annual limit."}`)) + return + } + } else if org.LeadInfo.BusinessLicenseCloud || org.LeadInfo.BusinessLicenseOnprem || org.LeadInfo.EnterpriseLicenseCloud || org.LeadInfo.EnterpriseLicenseOnprem || org.LeadInfo.ShuffleEnterpriseLicenseOldCustomer { + // Allow 1000% of monthly plan for enterprise and business plans + maxAllowed := org.SyncFeatures.AppExecutions.Limit * 10 + if tmpData.Billing.InternalAppRunsHardLimit > maxAllowed { + resp.WriteHeader(400) + resp.Write([]byte(`{"success": false, "reason": "Hard limit cannot exceed 1000% of the monthly limit."}`)) + return + } + } else { + // Don't allow the hard limit more than the monthly limit for scale plans + maxAllowed := org.SyncFeatures.AppExecutions.Limit + if tmpData.Billing.InternalAppRunsHardLimit > maxAllowed { + resp.WriteHeader(400) + resp.Write([]byte(`{"success": false, "reason": "Hard limit cannot exceed 1000% of the monthly limit."}`)) + return + } + } + + org.Billing.InternalAppRunsHardLimit = tmpData.Billing.InternalAppRunsHardLimit + } else { + // Allow any limit for the support users org.Billing.InternalAppRunsHardLimit = tmpData.Billing.InternalAppRunsHardLimit + } } //Update mfa required value @@ -14002,8 +14176,33 @@ func HandleEditOrg(resp http.ResponseWriter, request *http.Request) { if len(tmpData.LeadInfo) > 0 && user.SupportAccess { //log.Printf("[INFO] Updating lead info for %s to %s", org.Id, tmpData.LeadInfo) - // Make a new one, as to start with all from false - newLeadinfo := LeadInfo{} + newLeadinfo := org.LeadInfo + newLeadinfo.POV = false + newLeadinfo.ShuffleEnterpriseLicenseOldCustomer = false + newLeadinfo.ScaleLicenseCloudTrial = false + newLeadinfo.ScaleLicenseCloudCustomer = false + newLeadinfo.ScaleLicenseOnpremCustomer = false + newLeadinfo.BusinessLicenseCloud = false + newLeadinfo.BusinessLicenseOnprem = false + newLeadinfo.EnterpriseLicenseCloud = false + newLeadinfo.EnterpriseLicenseOnprem = false + newLeadinfo.IntegrationPartner = false + newLeadinfo.ServicePartner = false + newLeadinfo.ChannelPartner = false + newLeadinfo.TechPartner = false + newLeadinfo.Contacted = false + newLeadinfo.Lead = false + newLeadinfo.DemoDone = false + newLeadinfo.Customer = false + newLeadinfo.OldCustomer = false + newLeadinfo.OldLead = false + newLeadinfo.OpenSource = false + newLeadinfo.OpenSourceLicense = false + newLeadinfo.Internal = false + newLeadinfo.Student = false + newLeadinfo.Creator = false + newLeadinfo.TestingShuffle = false + newLeadinfo.DistributionPartner = false for _, lead := range tmpData.LeadInfo { if lead == "testing shuffle" || lead == "testing_shuffle" { @@ -14054,11 +14253,11 @@ func HandleEditOrg(resp http.ResponseWriter, request *http.Request) { newLeadinfo.Creator = true } - if lead == "tech partner" { + if lead == "tech partner" || lead == "Technology Partner" { newLeadinfo.TechPartner = true } - if lead == "integration partner" { + if lead == "integration partner" || lead == "Integration Partner" { newLeadinfo.IntegrationPartner = true } @@ -14066,17 +14265,258 @@ func HandleEditOrg(resp http.ResponseWriter, request *http.Request) { newLeadinfo.DistributionPartner = true } - if lead == "service partner" { + if lead == "service partner" || lead == "Service Partner" { newLeadinfo.ServicePartner = true } - if lead == "channel partner" { + if lead == "channel partner" || lead == "Channel Partner" { newLeadinfo.ChannelPartner = true } + + if lead == "Contacted" { + newLeadinfo.Contacted = true + } + + if lead == "Lead" { + newLeadinfo.Lead = true + } + + if lead == "Demo Done" { + newLeadinfo.DemoDone = true + } + + if lead == "Customer" { + newLeadinfo.Customer = true + } + + if lead == "Old Customer" { + newLeadinfo.OldCustomer = true + } + + if lead == "Old Lead" { + newLeadinfo.OldLead = true + } + + if lead == "Open Source" { + newLeadinfo.OpenSource = true + } + + if lead == "Open Source License" { + newLeadinfo.OpenSourceLicense = true + } + + if lead == "Internal" { + newLeadinfo.Internal = true + } + + if lead == "Sub Org" { + newLeadinfo.SubOrg = true + } + + if lead == "Student" { + newLeadinfo.Student = true + } + + if lead == "Creator" { + newLeadinfo.Creator = true + } + + if lead == "Testing Shuffle" { + newLeadinfo.TestingShuffle = true + } + + if lead == "Distribution Partner" { + newLeadinfo.DistributionPartner = true + } + + if lead == "POC License" { + newLeadinfo.POV = true + } + + if lead == "Enterprise License (Legacy)" { + newLeadinfo.ShuffleEnterpriseLicenseOldCustomer = true + } + + if lead == "Scale License Cloud Trial" { + newLeadinfo.ScaleLicenseCloudTrial = true + } + + if lead == "Scale License Cloud" { + newLeadinfo.ScaleLicenseCloudCustomer = true + } + + if lead == "Scale License Onprem" { + newLeadinfo.ScaleLicenseOnpremCustomer = true + } + + if lead == "Business License Cloud" { + newLeadinfo.BusinessLicenseCloud = true + } + + if lead == "Business License Onprem" { + newLeadinfo.BusinessLicenseOnprem = true + } + + if lead == "Enterprise License Cloud" { + newLeadinfo.EnterpriseLicenseCloud = true + } + + if lead == "Enterprise License Onprem" { + newLeadinfo.EnterpriseLicenseOnprem = true + } + } + + if newLeadinfo.ShuffleEnterpriseLicenseOldCustomer || + newLeadinfo.ScaleLicenseCloudCustomer || + newLeadinfo.ScaleLicenseOnpremCustomer || + newLeadinfo.BusinessLicenseCloud || + newLeadinfo.BusinessLicenseOnprem || + newLeadinfo.EnterpriseLicenseCloud || + newLeadinfo.EnterpriseLicenseOnprem { + newLeadinfo.Customer = true + } + + if newLeadinfo.ScaleLicenseOnpremCustomer || newLeadinfo.BusinessLicenseOnprem || newLeadinfo.EnterpriseLicenseOnprem { + newLeadinfo.OpenSource = true } org.LeadInfo = newLeadinfo + if newLeadinfo.EnterpriseLicenseOnprem || + newLeadinfo.BusinessLicenseOnprem { + + org.SyncFeatures.OnpremAppExecutions.Limit = 300000 + org.SyncFeatures.OnpremAppExecutions.Active = true + + org.SyncFeatures.Branding.Active = false + + org.SyncFeatures.AppExecutions.Limit = 2000 + + org.SyncFeatures.MultiEnv.Limit = 250 + org.SyncFeatures.MultiEnv.Active = true + + org.SyncFeatures.MultiTenant.Active = true + org.SyncFeatures.MultiTenant.Limit = 1000 + + org.SyncFeatures.SendSms.Active = true + org.SyncFeatures.SendMail.Active = true + + log.Printf("[INFO] Set limits to 300000 app runs / 250 envs / 1000 tenants for org %s (enterprise/business)", org.Id) + } else if newLeadinfo.ShuffleEnterpriseLicenseOldCustomer { + org.SyncFeatures.AppExecutions.Limit = 300000 + org.SyncFeatures.Branding.Active = false + + org.SyncFeatures.MultiEnv.Limit = 250 + org.SyncFeatures.MultiEnv.Active = true + + org.SyncFeatures.MultiTenant.Limit = 1000 + org.SyncFeatures.MultiTenant.Active = true + + org.LeadInfo.ScaleLicenseCloudTrial = false + + org.SyncFeatures.SendSms.Active = true + org.SyncFeatures.SendMail.Active = true + log.Printf("[INFO] Set limits to 300000 app runs / 250 envs / 1000 tenants for org %s (enterprise/business)", org.Id) + } else if newLeadinfo.EnterpriseLicenseCloud || + newLeadinfo.BusinessLicenseCloud { + + org.SyncFeatures.AppExecutions.Limit = 300000 + org.SyncFeatures.OnpremAppExecutions.Limit = 25000 + + org.LeadInfo.ScaleLicenseCloudTrial = false + + org.SyncFeatures.Branding.Active = false + org.SyncFeatures.MultiEnv.Active = true + org.SyncFeatures.MultiEnv.Limit = 250 + + org.SyncFeatures.MultiTenant.Active = true + org.SyncFeatures.MultiTenant.Limit = 1000 + + org.SyncFeatures.SendSms.Active = true + org.SyncFeatures.SendMail.Active = true + log.Printf("[INFO] Set limits to 300000 app runs / 250 envs / 1000 tenants for org %s (enterprise/business)", org.Id) + } else if newLeadinfo.POV { + org.SyncFeatures.AppExecutions.Limit = 10000 + org.SyncFeatures.Branding.Active = false + org.SyncFeatures.MultiEnv.Limit = 1 + org.SyncFeatures.MultiTenant.Limit = 3 + log.Printf("[INFO] Set limits to 10000 app runs / 1 env / 3 tenants for org %s (POC license)", org.Id) + } else if newLeadinfo.ScaleLicenseCloudTrial { + org.SyncFeatures.AppExecutions.Limit = 2000 + org.SyncFeatures.Branding.Active = false + org.SyncFeatures.MultiEnv.Limit = 1 + org.SyncFeatures.MultiTenant.Limit = 3 + log.Printf("[INFO] Set limits to 2000 app runs / 1 env / 3 tenants for org %s (Scale free trial)", org.Id) + } else if newLeadinfo.OpenSourceLicense { + org.SyncFeatures.OnpremAppExecutions.Active = true + org.SyncFeatures.OnpremAppExecutions.Limit = 25000 + org.SyncFeatures.Branding.Active = false + org.SyncFeatures.AppExecutions.Limit = 2000 + org.SyncFeatures.MultiEnv.Limit = 1 + org.SyncFeatures.MultiTenant.Limit = 3 + log.Printf("[INFO] Set onprem limits to 25K onprem app runs / 1 env / 3 tenants for org %s (Open Source License)", org.Id) + } else if newLeadinfo.IntegrationPartner || newLeadinfo.ServicePartner { + org.SyncFeatures.Branding.Active = true + } else { + org.SyncFeatures.AppExecutions.Limit = 2000 + org.SyncFeatures.OnpremAppExecutions.Limit = 25000 + org.SyncFeatures.OnpremAppExecutions.Active = true + org.SyncFeatures.MultiEnv.Limit = 1 + org.SyncFeatures.Branding.Active = false + org.SyncFeatures.MultiTenant.Limit = 3 + log.Printf("[INFO] Reset limits to defaults (2000 app runs / 1 env / 3 tenants) for org %s (no license)", org.Id) + } + + + if newLeadinfo.EnterpriseLicenseCloud || + newLeadinfo.EnterpriseLicenseOnprem || + newLeadinfo.ShuffleEnterpriseLicenseOldCustomer || + newLeadinfo.BusinessLicenseCloud || + newLeadinfo.BusinessLicenseOnprem || + newLeadinfo.ScaleLicenseOnpremCustomer || + newLeadinfo.ScaleLicenseCloudCustomer || + newLeadinfo.POV { + newLeadinfo.ScaleLicenseCloudTrial = false + org.LeadInfo.ScaleLicenseCloudTrial = false + } + + // Update active subscription name to match the new license status + subName := "" + if newLeadinfo.EnterpriseLicenseCloud { + subName = "Enterprise License (Cloud)" + } else if newLeadinfo.EnterpriseLicenseOnprem { + subName = "Enterprise License (OnPrem)" + } else if newLeadinfo.ShuffleEnterpriseLicenseOldCustomer { + subName = "Enterprise License (Legacy)" + } else if newLeadinfo.BusinessLicenseCloud { + subName = "Business License (Cloud)" + } else if newLeadinfo.BusinessLicenseOnprem { + subName = "Business License (OnPrem)" + } else if newLeadinfo.ScaleLicenseOnpremCustomer { + subName = "Scale License (OnPrem)" + } else if newLeadinfo.ScaleLicenseCloudCustomer { + subName = "Scale License (Cloud)" + } else if newLeadinfo.ScaleLicenseCloudTrial { + subName = "Scale License (Cloud Trial)" + } else if newLeadinfo.POV { + subName = "POC License (Limited Period)" + } + if subName != "" { + isAnnualPlan := strings.Contains(subName, "Business") || strings.Contains(subName, "Enterprise") + for i := range org.Subscriptions { + if org.Subscriptions[i].Active { + org.Subscriptions[i].Name = subName + if isAnnualPlan { + if org.Subscriptions[i].Startdate == 0 { + org.Subscriptions[i].Startdate = time.Now().Unix() + } + org.Subscriptions[i].Enddate = org.Subscriptions[i].Startdate + 365*24*60*60 + } + } + } + log.Printf("[INFO] Updated active subscription name to %s for org %s", subName, org.Id) + } + // Check for ORG_CHANGE_WEBHOOK orgWebhook := os.Getenv("ORG_CHANGE_WEBHOOK") if orgWebhook != "" && strings.HasPrefix(orgWebhook, "http") { @@ -14238,7 +14678,7 @@ func HandleEditOrg(resp http.ResponseWriter, request *http.Request) { } // check if user is editing sync features of suborg from parent org - if project.Environment == "cloud" && !user.SupportAccess && tmpData.SyncFeatures.Editing && tmpData.Editing != "app_runs_hard_limit" { + if project.Environment == "cloud" && !user.SupportAccess && tmpData.SyncFeatures.Editing && tmpData.Editing != "app_runs_hard_limit" && tmpData.Editing != "app_runs_grouping" { log.Printf("[WARNING] User %s (%s) is trying to edit sync features of suborg %s (%s)", user.Username, user.Id, org.Name, org.Id) // check whether user org id is suborg of parent org @@ -14374,6 +14814,46 @@ func HandleEditOrg(resp http.ResponseWriter, request *http.Request) { } +func SendLicenseExpiredRequest(orgId string, apikey string) { + log.Printf("[INFO] Subscription expired for org %s, sending license_expired update", orgId) + url := fmt.Sprintf("https://shuffler.io/api/v1/orgs/%s", orgId) + payloadData := map[string]string{ + "editing": "license_expired", + "org_id": orgId, + } + payload, _ := json.Marshal(payloadData) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(payload)) + if err != nil { + log.Printf("[ERROR] Failed to create request for license_expired: %v", err) + return + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Org-Id", orgId) + if apikey != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apikey)) + } + + client := &http.Client{ + Timeout: 10 * time.Second, + } + res, err := client.Do(req) + if err != nil { + log.Printf("[ERROR] Failed to send license_expired request to edit org %s: %v", orgId, err) + return + } + defer res.Body.Close() + + if res.StatusCode != 200 { + log.Printf("[WARNING] license_expired request returned status code %d for org %s", res.StatusCode, orgId) + } else { + log.Printf("[INFO] Successfully marked license as expired for org %s", orgId) + } +} + func sendMailSendgrid(toEmail []string, subject, body string, emailApp bool, BccAddresses []string) error { log.Printf("[DEBUG] In mail sending with subject %s and body length %s. TO: %s", subject, body, toEmail) srequest := sendgrid.GetRequest(os.Getenv("SENDGRID_API_KEY"), "/v3/mail/send", "https://api.sendgrid.com") @@ -33195,6 +33675,26 @@ func HandleDeleteOrg(resp http.ResponseWriter, request *http.Request) { return } + orgStats, err := GetOrgStatistics(ctx, parentOrg.Id) + if err == nil && orgStats != nil { + newTenant := Tenants{ + Name: subOrg.Name, + Id: subOrg.Id, + CreatedAt: time.Unix(subOrg.Created, 0), + DeletedAt: time.Now(), + Status: "deleted", + } + + if orgStats.Tenants == nil { + orgStats.Tenants = []Tenants{} + } + orgStats.Tenants = append(orgStats.Tenants, newTenant) + err = SetOrgStatistics(ctx, *orgStats, parentOrg.Id) + if err != nil { + log.Printf("[WARNING] Failed setting org statistics for org '%s': %s", parentOrg.Id, err) + } + } + user.Orgs = newOrgString if user.ActiveOrg.Id == subOrg.Id { // If the user is in the org that was deleted, set active org as parent org @@ -33898,6 +34398,8 @@ func HandleCheckLicense(ctx context.Context, org Org) Org { org.SyncFeatures.AppExecutions.Active = false org.SyncFeatures.AppExecutions.Limit = 25000 + org.SyncFeatures.AnnualAppRunsGrouping.Active = false + return org } features := SyncFeatures{} @@ -33919,6 +34421,8 @@ func HandleCheckLicense(ctx context.Context, org Org) Org { org.SyncFeatures.Branding.Active = features.Branding.Active + org.SyncFeatures.AnnualAppRunsGrouping.Active = features.AnnualAppRunsGrouping.Active + org.SyncFeatures.AppExecutions.Active = features.OnpremAppExecutions.Active if features.OnpremAppExecutions.Limit < 25000 { org.SyncFeatures.AppExecutions.Limit = 25000 @@ -33935,6 +34439,7 @@ func HandleCheckLicense(ctx context.Context, org Org) Org { org.SyncFeatures.Branding.Active = false org.SyncFeatures.AppExecutions.Active = false org.SyncFeatures.AppExecutions.Limit = 25000 + org.SyncFeatures.AnnualAppRunsGrouping.Active = false } } else { org.Licensed = false @@ -33947,6 +34452,7 @@ func HandleCheckLicense(ctx context.Context, org Org) Org { org.SyncFeatures.Branding.Active = false org.SyncFeatures.AppExecutions.Active = false org.SyncFeatures.AppExecutions.Limit = 25000 + org.SyncFeatures.AnnualAppRunsGrouping.Active = false } @@ -33957,6 +34463,8 @@ func HandleCheckLicense(ctx context.Context, org Org) Org { org.SyncFeatures.AppExecutions.Limit = features.OnpremAppExecutions.Limit } + org.SyncFeatures.AnnualAppRunsGrouping.Active = features.AnnualAppRunsGrouping.Active + org.SyncFeatures.Webhook.Active = features.Webhook.Active org.SyncFeatures.Webhook.Limit = features.Webhook.Limit @@ -34016,6 +34524,7 @@ func HandleCheckLicense(ctx context.Context, org Org) Org { org.SyncFeatures.Branding.Active = false org.SyncFeatures.AppExecutions.Active = false + org.SyncFeatures.AnnualAppRunsGrouping.Active = false org.SyncFeatures.AppExecutions.Limit = 25000 } @@ -34039,6 +34548,7 @@ func HandleCheckLicense(ctx context.Context, org Org) Org { } org.SyncFeatures.Branding.Active = license.Branding + org.SyncFeatures.AnnualAppRunsGrouping.Active = license.AppRunsGrouping } } @@ -34056,6 +34566,25 @@ func HandleCheckLicense(ctx context.Context, org Org) Org { } } + appRunsHardLimitCacheKey := fmt.Sprintf("org_app_runs_hard_limit_%s", org.Id) + appRunsHardLimit, err := GetCache(ctx, appRunsHardLimitCacheKey) + if err != nil { + log.Printf("[ERROR] Failed to get cache for org (%s) subscriptions in HandleCheckLicense: %v", org.Id, err) + } else { + if appRunsHardLimit != nil { + if data, ok := appRunsHardLimit.([]byte); ok { + var limit int64 + if err := json.Unmarshal(data, &limit); err == nil { + org.Billing.InternalAppRunsHardLimit = limit + } else if parsedLimit, err := strconv.ParseInt(string(data), 10, 64); err == nil { + org.Billing.InternalAppRunsHardLimit = parsedLimit + } + } else if limit, ok := appRunsHardLimit.(int64); ok { + org.Billing.InternalAppRunsHardLimit = limit + } + } + } + } else if len(shuffleLicenseKey) > 0 { license := checkNoInternet() @@ -34071,6 +34600,7 @@ func HandleCheckLicense(ctx context.Context, org Org) Org { org.SyncFeatures.Branding.Active = license.Branding org.SyncFeatures.AppExecutions.Active = license.AppRuns.Active org.SyncFeatures.AppExecutions.Limit = license.AppRuns.Limit + org.SyncFeatures.AnnualAppRunsGrouping.Active = license.AppRunsGrouping org.SyncFeatures.WorkflowExecutions.Active = true org.SyncFeatures.Webhook.Active = true @@ -34099,6 +34629,7 @@ func HandleCheckLicense(ctx context.Context, org Org) Org { org.SyncFeatures.AppExecutions.Active = false org.SyncFeatures.AppExecutions.Limit = 25000 + org.SyncFeatures.AnnualAppRunsGrouping.Active = false } parsedEula := GetOnpremPaidEula() @@ -34109,6 +34640,7 @@ func HandleCheckLicense(ctx context.Context, org Org) Org { var endDate int64 var cancellationDate int64 + var startDate int64 active := false features := []string{ @@ -34133,22 +34665,35 @@ func HandleCheckLicense(ctx context.Context, org Org) Org { parsedTimeout = time.Now() } endDate = parsedTimeout.Unix() + + parsedStartDate, err := time.Parse("02-01-2006", license.StartDate) + if err != nil { + parsedStartDate = time.Now() + } + startDate = parsedStartDate.Unix() + cancellationDate = 0 active = true } else { endDate = time.Now().Unix() + startDate = time.Now().Unix() cancellationDate = time.Now().Unix() active = false } + recurrance := string("monthly") + + if license.AppRunsGrouping { + recurrance = string("annual") + } subscription := PaymentSubscription{ Name: "Enterprise License", Active: active, CancellationDate: cancellationDate, SupportLevel: "Enterprise Support", - Startdate: time.Now().Unix(), + Startdate: startDate, Enddate: endDate, - Recurrence: string("monthly"), + Recurrence: recurrance, Amount: "0", Currency: string("USD"), Level: "1", @@ -34177,6 +34722,8 @@ func HandleCheckLicense(ctx context.Context, org Org) Org { org.SyncFeatures.AppExecutions.Active = false org.SyncFeatures.AppExecutions.Limit = 25000 + + org.SyncFeatures.AnnualAppRunsGrouping.Active = false } return org diff --git a/stats.go b/stats.go index 53d5794b..09a13285 100755 --- a/stats.go +++ b/stats.go @@ -9,6 +9,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" "encoding/json" @@ -653,6 +654,50 @@ func GetSpecificStats(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": %v, "key": "%s", "total": %d, "available_keys": %s, "entries": %s}`, successful, strings.ReplaceAll(statsKey, "\"", ""), totalValue, string(availableStats), string(marshalledEntries)))) } +func mergeMultiRegionResults(crossRegionResults []MultiRegionStatsEntry, info *ExecutionInfo, parentDailyMap map[string]int) { + log.Printf("[INFO] HandleGetStatistics: received cross-region stats for %d child orgs from multi-region-stats endpoint", len(crossRegionResults)) + + for _, entry := range crossRegionResults { + for _, childDay := range entry.DailyStatistics { + dateKey := childDay.Date.UTC().Format("2006-01-02") + alreadyAppliedCorrection := childDay.AgentInputTokens*250/1_000_000 + + childDay.AgentOutputTokens*1500/1_000_000 + + childDay.DailySMSUsage*3 + + childDay.DailyEmailUsage*2 + rawChildAppExecutions := childDay.AppExecutions - alreadyAppliedCorrection + if rawChildAppExecutions < 0 { + rawChildAppExecutions = 0 + } + + if idx, exists := parentDailyMap[dateKey]; exists { + info.DailyStatistics[idx].ChildAppExecutions += rawChildAppExecutions + info.DailyStatistics[idx].DailyChildOrgAiUsage += childDay.AIUsage + info.DailyStatistics[idx].DailyChildOrgAgentExecutions += childDay.AgentExecutions + info.DailyStatistics[idx].DailyChildOrgAgentTokens += childDay.AgentTokens + info.DailyStatistics[idx].DailyChildOrgAgentInputTokens += childDay.AgentInputTokens + info.DailyStatistics[idx].DailyChildOrgAgentOutputTokens += childDay.AgentOutputTokens + info.DailyStatistics[idx].DailyChildOrgSMSUsage += childDay.DailySMSUsage + info.DailyStatistics[idx].DailyChildOrgEmailUsage += childDay.DailyEmailUsage + } else { + // No matching parent day — create a new entry carrying only the child org counters + newDay := DailyStatistics{ + Date: childDay.Date, + ChildAppExecutions: rawChildAppExecutions, + DailyChildOrgAiUsage: childDay.AIUsage, + DailyChildOrgAgentExecutions: childDay.AgentExecutions, + DailyChildOrgAgentTokens: childDay.AgentTokens, + DailyChildOrgAgentInputTokens: childDay.AgentInputTokens, + DailyChildOrgAgentOutputTokens: childDay.AgentOutputTokens, + DailyChildOrgSMSUsage: childDay.DailySMSUsage, + DailyChildOrgEmailUsage: childDay.DailyEmailUsage, + } + parentDailyMap[dateKey] = len(info.DailyStatistics) + info.DailyStatistics = append(info.DailyStatistics, newDay) + } + } + } +} + func HandleGetStatistics(resp http.ResponseWriter, request *http.Request) { cors := HandleCors(resp, request) if cors { @@ -684,7 +729,7 @@ func HandleGetStatistics(resp http.ResponseWriter, request *http.Request) { org := &Org{} ctx := GetContext(request) - if orgId == "public" { + if orgId == "public" { if user.SupportAccess { log.Printf("[AUDIT] User %s (%s) is getting org stats for PUBLIC org %s with support access", user.Username, user.Id, orgId) } @@ -833,6 +878,54 @@ func HandleGetStatistics(resp http.ResponseWriter, request *http.Request) { } } + key = fmt.Sprintf("cache_%s_send_mail", orgId) + cacheItem, err = GetCache(ctx, key) + if err == nil { + parsedItem := []byte(cacheItem.([]uint8)) + increment, err := strconv.Atoi(string(parsedItem)) + if err == nil { + info.TotalEmailUsage += int64(increment) + info.MonthlyEmailUsage += int64(increment) + info.DailyEmailUsage += int64(increment) + } + } + + key = fmt.Sprintf("cache_%s_childorg_send_mail", orgId) + cacheItem, err = GetCache(ctx, key) + if err == nil { + parsedItem := []byte(cacheItem.([]uint8)) + increment, err := strconv.Atoi(string(parsedItem)) + if err == nil { + info.TotalChildOrgEmailUsage += int64(increment) + info.MonthlyChildOrgEmailUsage += int64(increment) + info.DailyChildOrgEmailUsage += int64(increment) + } + } + + key = fmt.Sprintf("cache_%s_send_sms", orgId) + cacheItem, err = GetCache(ctx, key) + if err == nil { + parsedItem := []byte(cacheItem.([]uint8)) + increment, err := strconv.Atoi(string(parsedItem)) + if err == nil { + info.TotalSMSUsage += int64(increment) + info.MonthlySMSUsage += int64(increment) + info.DailySMSUsage += int64(increment) + } + } + + key = fmt.Sprintf("cache_%s_childorg_send_sms", orgId) + cacheItem, err = GetCache(ctx, key) + if err == nil { + parsedItem := []byte(cacheItem.([]uint8)) + increment, err := strconv.Atoi(string(parsedItem)) + if err == nil { + info.TotalChildOrgSMSUsage += int64(increment) + info.MonthlyChildOrgSMSUsage += int64(increment) + info.DailyChildOrgSMSUsage += int64(increment) + } + } + for additionCnt, addition := range info.Additions { key := fmt.Sprintf("cache_%s_%s", orgId, addition.Key) @@ -868,7 +961,207 @@ func HandleGetStatistics(resp http.ResponseWriter, request *http.Request) { } } - newjson, err := json.Marshal(info) + parentOrgLocations := []Locations{} + parentEnvs, err := GetEnvironments(ctx, org.Id) + if err == nil { + for _, env := range parentEnvs { + if strings.ToLower(env.Name) == "cloud" { + continue + } + status := "active" + if env.Archived { + status = "disabled" + } + parentOrgLocations = append(parentOrgLocations, Locations{ + OrgId: org.Id, + OrgName: org.Name, + Name: env.Name, + Id: env.Id, + CreatedAt: time.Unix(env.Created, 0).Format(time.RFC3339), + Status: status, + }) + } + } + + if len(parentOrgLocations) > 0 { + info.Locations = append(parentOrgLocations, info.Locations...) + } + + skipMultiRegion := false + if skipList, ok := request.URL.Query()["skip_multi_region"]; ok && len(skipList) > 0 && skipList[0] == "true" { + skipMultiRegion = true + } + + if len(org.CreatorOrg) > 0 { + skipMultiRegion = true + } + + if len(org.ChildOrgs) > 0 && !skipMultiRegion { + // Build a date-keyed map of parent daily stats for fast lookups when merging cross-region child data + parentDailyMap := make(map[string]int, len(info.DailyStatistics)) + for i, d := range info.DailyStatistics { + parentDailyMap[d.Date.UTC().Format("2006-01-02")] = i + } + + // mu protects all shared writes: info.Tenants, info.Locations, info.DailyStatistics, parentDailyMap + var mu sync.Mutex + var wg sync.WaitGroup + + // Semaphore: buffered channel of size 5 limits concurrent goroutines to 5 at a time + sem := make(chan struct{}, 5) + + parentRegionUrl := strings.TrimRight(org.RegionUrl, "/") + hasCrossRegionChildren := false + + for _, childOrgMini := range org.ChildOrgs { + wg.Add(1) + sem <- struct{}{} // acquire a slot; blocks if 5 goroutines are already running + + go func(childOrgMini OrgMini) { + defer wg.Done() + defer func() { <-sem }() // release the slot when this goroutine finishes + + childOrgFull, err := GetOrg(ctx, childOrgMini.Id) + if err != nil { + log.Printf("[WARNING] HandleGetStatistics: failed fetching child org %s: %s", childOrgMini.Id, err) + return + } + + // --- Collect Tenant and Location data (write behind mutex) --- + newTenant := Tenants{ + Name: childOrgFull.Name, + Id: childOrgFull.Id, + CreatedAt: time.Unix(childOrgFull.Created, 0), + Status: "active", + } + + var newLocs []Locations + childEnvs, err := GetEnvironments(ctx, childOrgFull.Id) + if err == nil { + for _, env := range childEnvs { + if len(env.SuborgDistribution) > 0 { + continue + } + if strings.ToLower(env.Name) == "cloud" { + continue + } + status := "active" + if env.Archived { + status = "disabled" + } + newLocs = append(newLocs, Locations{ + OrgId: childOrgFull.Id, + OrgName: childOrgFull.Name, + Name: env.Name, + Id: env.Id, + CreatedAt: time.Unix(env.Created, 0).Format(time.RFC3339), + Status: status, + }) + } + } + + childRegionUrl := strings.TrimRight(childOrgFull.RegionUrl, "/") + if project.Environment == "cloud" && + len(childRegionUrl) > 0 && + strings.Contains(childRegionUrl, "http") && + childRegionUrl != parentRegionUrl { + mu.Lock() + hasCrossRegionChildren = true + info.Tenants = append(info.Tenants, newTenant) + info.Locations = append(info.Locations, newLocs...) + mu.Unlock() + return + } + + mu.Lock() + info.Tenants = append(info.Tenants, newTenant) + info.Locations = append(info.Locations, newLocs...) + mu.Unlock() + }(childOrgMini) + } + + wg.Wait() + + if project.Environment == "cloud" && hasCrossRegionChildren && !skipMultiRegion { + multiRegionCacheKey := fmt.Sprintf("multi_region_stats_%s", org.Id) + + if cachedBody, cacheErr := GetCache(ctx, multiRegionCacheKey); cacheErr == nil { + cachedBytes := []byte(cachedBody.([]uint8)) + var cachedResults []MultiRegionStatsEntry + if jsonErr := json.Unmarshal(cachedBytes, &cachedResults); jsonErr == nil { + log.Printf("[INFO] HandleGetStatistics: serving multi-region stats from cache for org %s", orgId) + mergeMultiRegionResults(cachedResults, info, parentDailyMap) + } else { + log.Printf("[WARNING] HandleGetStatistics: failed unmarshalling cached multi-region-stats, will refetch: %s", jsonErr) + } + } else { + multiRegionUrl := fmt.Sprintf("https://shuffler.io/api/v1/orgs/%s/multi-region-stats", orgId) + multiReq, multiErr := http.NewRequest("GET", multiRegionUrl, nil) + if multiErr != nil { + log.Printf("[WARNING] HandleGetStatistics: failed building multi-region-stats request: %s", multiErr) + } else { + multiReq.Header.Set("Authorization", fmt.Sprintf("Bearer %s", user.ApiKey)) + multiReq.Header.Set("Org-Id", orgId) + + multiClient := &http.Client{Timeout: 60 * time.Second} + multiResp, multiDoErr := multiClient.Do(multiReq) + if multiDoErr != nil { + log.Printf("[WARNING] HandleGetStatistics: multi-region-stats request failed: %s", multiDoErr) + } else { + defer multiResp.Body.Close() + if multiResp.StatusCode == 200 { + multiBody, multiReadErr := ioutil.ReadAll(multiResp.Body) + if multiReadErr != nil { + log.Printf("[WARNING] HandleGetStatistics: failed reading multi-region-stats body: %s", multiReadErr) + } else { + var crossRegionResults []MultiRegionStatsEntry + if jsonErr := json.Unmarshal(multiBody, &crossRegionResults); jsonErr != nil { + log.Printf("[WARNING] HandleGetStatistics: failed unmarshalling multi-region-stats: %s", jsonErr) + } else { + // Store raw response bytes in cache for 1 hour (3600 seconds) + _ = SetCache(ctx, multiRegionCacheKey, multiBody, 3600) + mergeMultiRegionResults(crossRegionResults, info, parentDailyMap) + } + } + } else { + log.Printf("[WARNING] HandleGetStatistics: multi-region-stats returned status %d", multiResp.StatusCode) + } + } + } + } + } + } + + if org.SyncFeatures.AnnualAppRunsGrouping.Active { + var startDate, endDate time.Time + annualSubscriptionExists := false + for _, subscription := range org.Subscriptions { + if (strings.Contains(strings.ToLower(subscription.Name), "business") || strings.Contains(strings.ToLower(subscription.Name), "enterprise")) && (strings.Contains(strings.ToLower(subscription.Recurrence), "annual")) && subscription.Active { + annualSubscriptionExists = true + startDate = time.Unix(subscription.Startdate, 0) + endDate = time.Unix(subscription.Enddate, 0) + break + } + } + + if annualSubscriptionExists && len(info.DailyStatistics) > 0 { + + annualSubscriptionAppRuns := int64(0) + annualSubscriptionChildAppRuns := int64(0) + for i := range info.DailyStatistics { + if info.DailyStatistics[i].Date.Unix() >= startDate.Unix() && info.DailyStatistics[i].Date.Unix() <= endDate.Unix() { + annualSubscriptionAppRuns += info.DailyStatistics[i].AppExecutions + annualSubscriptionChildAppRuns += info.DailyStatistics[i].ChildAppExecutions + } + } + info.AnnualAppExecutions = annualSubscriptionAppRuns + info.AnnualChildAppExecutions = annualSubscriptionChildAppRuns + } + } + + stats := GetCorrectedStats(info) + + newjson, err := json.Marshal(stats) if err != nil { log.Printf("[ERROR] Failed marshal in get org stats: %s", err) resp.WriteHeader(500) @@ -880,6 +1173,56 @@ func HandleGetStatistics(resp http.ResponseWriter, request *http.Request) { resp.Write(newjson) } +// Make sure that we are not calling SetOrgStatistics function after calling this function. This will increase the app runs count in db on every call to this function. +func GetCorrectedStats(info *ExecutionInfo) *ExecutionInfo { + + // 1 Million Input Tokens = 250 app runs + // 1 Million Output Tokens = 1500 app runs + // 1 SMS = 3 app runs + // 1 Email = 2 app runs + + // Loop through the daily statistics and add the app runs from tokens, SMS and email on top of existing counts + for i := range info.DailyStatistics { + info.DailyStatistics[i].AppExecutions += info.DailyStatistics[i].AgentInputTokens*250/1_000_000 + info.DailyStatistics[i].AgentOutputTokens*1500/1_000_000 + info.DailyStatistics[i].DailySMSUsage*3 + info.DailyStatistics[i].DailyEmailUsage*2 + info.DailyStatistics[i].ChildAppExecutions += info.DailyStatistics[i].ChildOrgAgentInputTokens*250/1_000_000 + info.DailyStatistics[i].ChildOrgAgentOutputTokens*1500/1_000_000 + info.DailyStatistics[i].DailyChildOrgSMSUsage*3 + info.DailyStatistics[i].DailyChildOrgEmailUsage*2 + } + + // Add the monthly app runs from SMS, Email, Input Tokens and Output Tokens on top of existing counts + info.MonthlyAppExecutions += info.MonthlySMSUsage*3 + info.MonthlyEmailUsage*2 + info.MonthlyAgentInputTokens*250/1_000_000 + info.MonthlyAgentOutputTokens*1500/1_000_000 + info.MonthlyChildAppExecutions += info.MonthlyChildOrgSMSUsage*3 + info.MonthlyChildOrgEmailUsage*2 + info.MonthlyChildOrgAgentInputTokens*250/1_000_000 + info.MonthlyChildOrgAgentOutputTokens*1500/1_000_000 + + info.DailyAppExecutions += info.DailyAgentInputTokens*250/1_000_000 + info.DailyAgentOutputTokens*1500/1_000_000 + info.DailySMSUsage*3 + info.DailyEmailUsage*2 + info.DailyChildAppExecutions += info.DailyChildOrgAgentInputTokens*250/1_000_000 + info.DailyChildOrgAgentOutputTokens*1500/1_000_000 + info.DailyChildOrgSMSUsage*3 + info.DailyChildOrgEmailUsage*2 + + info.TotalAppExecutions += info.TotalAgentInputTokens*250/1_000_000 + info.TotalAgentOutputTokens*1500/1_000_000 + info.TotalSMSUsage*3 + info.TotalEmailUsage*2 + info.TotalChildAppExecutions += info.TotalChildOrgAgentInputTokens*250/1_000_000 + info.TotalChildOrgAgentOutputTokens*1500/1_000_000 + info.TotalChildOrgSMSUsage*3 + info.TotalChildOrgEmailUsage*2 + + if len(info.DailyStatistics) > 0 { + annualInputTokens := int64(0) + annualChildInputTokens := int64(0) + annualOutputTokens := int64(0) + annualChildOutputTokens := int64(0) + annualSMSUsage := int64(0) + annualChildSMSUsage := int64(0) + annualEmailUsage := int64(0) + annualChildEmailUsage := int64(0) + for i := range info.DailyStatistics { + annualInputTokens += info.DailyStatistics[i].AgentInputTokens + annualChildInputTokens += info.DailyStatistics[i].ChildOrgAgentInputTokens + annualOutputTokens += info.DailyStatistics[i].AgentOutputTokens + annualChildOutputTokens += info.DailyStatistics[i].ChildOrgAgentOutputTokens + annualSMSUsage += info.DailyStatistics[i].DailySMSUsage + annualChildSMSUsage += info.DailyStatistics[i].DailyChildOrgSMSUsage + annualEmailUsage += info.DailyStatistics[i].DailyEmailUsage + annualChildEmailUsage += info.DailyStatistics[i].DailyChildOrgEmailUsage + } + info.AnnualAppExecutions += annualInputTokens*250/1_000_000 + annualOutputTokens*1500/1_000_000 + annualSMSUsage*3 + annualEmailUsage*2 + info.AnnualChildAppExecutions += annualChildInputTokens*250/1_000_000 + annualChildOutputTokens*1500/1_000_000 + annualChildSMSUsage*3 + annualChildEmailUsage*2 + } + + return info +} + func HandleAppendStatistics(resp http.ResponseWriter, request *http.Request) { // Send in a thing to increment cors := HandleCors(resp, request) @@ -946,7 +1289,6 @@ func HandleAppendStatistics(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Cache incremented by %d"}`, inputData.Value))) } - // Rudementary caching system. WILL go wrong at times without sharding. // It's only good for the user in cloud, hence wont bother for a while // Optional input is the amount to increment @@ -1317,104 +1659,118 @@ func IncrementCache(ctx context.Context, orgId, dataType string, amount ...int) // 2. If there isn't, set it and clear out the daily records // Also: can we dump a list of apps that run? Maybe a list of them? func handleDailyCacheUpdate(executionInfo *ExecutionInfo) *ExecutionInfo { - timeYesterday := time.Now().AddDate(0, 0, -1) - timeYesterdayFormatted := timeYesterday.Format("2006-12-02") - - for _, day := range executionInfo.DailyStatistics { - - // Check if the day.Date is the same as yesterday and return if it is - if day.Date.Format("2006-12-02") == timeYesterdayFormatted { - for additionIndex, _ := range executionInfo.Additions { - executionInfo.Additions[additionIndex].DailyValue = 0 - } - - return executionInfo - } - } - - log.Printf("[DEBUG] Daily stats not updated for %s in org %s today. Only have %d stats so far - running update.", timeYesterday, executionInfo.OrgId, len(executionInfo.DailyStatistics)) - // If we get here, we need to update the daily stats - newDay := DailyStatistics{ - Date: timeYesterday, - AppExecutions: executionInfo.DailyAppExecutions, - ChildAppExecutions: executionInfo.DailyChildAppExecutions, - AppExecutionsFailed: executionInfo.DailyAppExecutionsFailed, - SubflowExecutions: executionInfo.DailySubflowExecutions, - WorkflowExecutions: executionInfo.DailyWorkflowExecutions, - WorkflowExecutionsFinished: executionInfo.DailyWorkflowExecutionsFinished, - WorkflowExecutionsFailed: executionInfo.DailyWorkflowExecutionsFailed, - OrgSyncActions: executionInfo.DailyOrgSyncActions, - CloudExecutions: executionInfo.DailyCloudExecutions, - OnpremExecutions: executionInfo.DailyOnpremExecutions, - AIUsage: executionInfo.DailyAIUsage, - AgentExecutions: executionInfo.DailyAgentExecutions, - AgentExecutionsSuccessful: executionInfo.DailyAgentExecutionsSuccessful, - AgentExecutionsFailed: executionInfo.DailyAgentExecutionsFailed, - AgentTokens: executionInfo.DailyAgentTokens, - AgentInputTokens: executionInfo.DailyAgentInputTokens, - AgentOutputTokens: executionInfo.DailyAgentOutputTokens, - AgentCachedTokens: executionInfo.DailyAgentCachedTokens, - - ApiUsage: executionInfo.DailyApiUsage, - - Additions: executionInfo.Additions, - } - - executionInfo.DailyStatistics = append(executionInfo.DailyStatistics, newDay) - - // Cleaning up old stuff we don't use for now - executionInfo.HourlyAppExecutions = 0 - executionInfo.HourlyChildAppExecutions = 0 - executionInfo.HourlyAppExecutionsFailed = 0 - executionInfo.HourlySubflowExecutions = 0 - executionInfo.HourlyWorkflowExecutions = 0 - executionInfo.HourlyWorkflowExecutionsFinished = 0 - executionInfo.HourlyChildWorkflowExecutions = 0 - executionInfo.HourlyWorkflowExecutionsFailed = 0 - executionInfo.HourlyOrgSyncActions = 0 - executionInfo.HourlyCloudExecutions = 0 - executionInfo.HourlyOnpremExecutions = 0 - - // Reset daily - executionInfo.DailyAppExecutions = 0 - executionInfo.DailyChildAppExecutions = 0 - executionInfo.DailyAppExecutionsFailed = 0 - executionInfo.DailySubflowExecutions = 0 - executionInfo.DailyWorkflowExecutions = 0 - executionInfo.DailyWorkflowExecutionsFinished = 0 - executionInfo.DailyChildWorkflowExecutions = 0 - executionInfo.DailyWorkflowExecutionsFailed = 0 - executionInfo.DailyOrgSyncActions = 0 - executionInfo.DailyCloudExecutions = 0 - executionInfo.DailyOnpremExecutions = 0 - executionInfo.DailyApiUsage = 0 - executionInfo.DailyAIUsage = 0 - executionInfo.DailyAgentExecutions = 0 - executionInfo.DailyAgentExecutionsSuccessful = 0 - executionInfo.DailyAgentExecutionsFailed = 0 - executionInfo.DailyAgentMaxLoopsHit = 0 - executionInfo.DailyAgentTokens = 0 - executionInfo.DailyAgentInputTokens = 0 - executionInfo.DailyAgentOutputTokens = 0 - executionInfo.DailyAgentCachedTokens = 0 - - // Weekly - executionInfo.WeeklyAppExecutions = 0 - executionInfo.WeeklyChildAppExecutions = 0 - executionInfo.WeeklyAppExecutionsFailed = 0 - executionInfo.WeeklySubflowExecutions = 0 - executionInfo.WeeklyWorkflowExecutions = 0 - executionInfo.WeeklyWorkflowExecutionsFinished = 0 - executionInfo.WeeklyWorkflowExecutionsFailed = 0 - executionInfo.WeeklyOrgSyncActions = 0 - executionInfo.WeeklyCloudExecutions = 0 - executionInfo.WeeklyOnpremExecutions = 0 - executionInfo.WeeklyChildWorkflowExecutions = 0 - - // Cleans up "random" stats as well - for additionIndex, _ := range executionInfo.Additions { - executionInfo.Additions[additionIndex].Value = 0 - executionInfo.Additions[additionIndex].DailyValue = 0 + currentDate := time.Now().Format("2006-01-02") + + // check if today's date exists in daily stats, if not (new day), append it and reset daily values + if len(executionInfo.DailyStatistics) == 0 || executionInfo.DailyStatistics[len(executionInfo.DailyStatistics)-1].Date.Format("2006-01-02") != currentDate { + executionInfo.DailyStatistics = append(executionInfo.DailyStatistics, DailyStatistics{ + Date: time.Now(), + }) + + // Reset daily and hourly/weekly fields so they start fresh + executionInfo.HourlyAppExecutions = 0 + executionInfo.HourlyChildAppExecutions = 0 + executionInfo.HourlyAppExecutionsFailed = 0 + executionInfo.HourlySubflowExecutions = 0 + executionInfo.HourlyWorkflowExecutions = 0 + executionInfo.HourlyWorkflowExecutionsFinished = 0 + executionInfo.HourlyChildWorkflowExecutions = 0 + executionInfo.HourlyWorkflowExecutionsFailed = 0 + executionInfo.HourlyOrgSyncActions = 0 + executionInfo.HourlyCloudExecutions = 0 + executionInfo.HourlyOnpremExecutions = 0 + + executionInfo.DailyAppExecutions = 0 + executionInfo.DailyChildAppExecutions = 0 + executionInfo.DailyAppExecutionsFailed = 0 + executionInfo.DailySubflowExecutions = 0 + executionInfo.DailyWorkflowExecutions = 0 + executionInfo.DailyWorkflowExecutionsFinished = 0 + executionInfo.DailyChildWorkflowExecutions = 0 + executionInfo.DailyWorkflowExecutionsFailed = 0 + executionInfo.DailyOrgSyncActions = 0 + executionInfo.DailyCloudExecutions = 0 + executionInfo.DailyOnpremExecutions = 0 + executionInfo.DailyApiUsage = 0 + executionInfo.DailyAIUsage = 0 + executionInfo.DailyAgentExecutions = 0 + executionInfo.DailyAgentTokens = 0 + executionInfo.DailyAgentInputTokens = 0 + executionInfo.DailyAgentOutputTokens = 0 + executionInfo.DailyChildOrgAiUsage = 0 + executionInfo.DailyChildOrgAgentExecutions = 0 + executionInfo.DailyChildOrgAgentTokens = 0 + executionInfo.DailyChildOrgAgentInputTokens = 0 + executionInfo.DailyChildOrgAgentOutputTokens = 0 + executionInfo.DailySMSUsage = 0 + executionInfo.DailyChildOrgSMSUsage = 0 + executionInfo.DailyEmailUsage = 0 + executionInfo.DailyChildOrgEmailUsage = 0 + executionInfo.DailyAgentExecutionsSuccessful = 0 + executionInfo.DailyAgentExecutionsFailed = 0 + executionInfo.DailyAgentCachedTokens = 0 + executionInfo.DailyAgentMaxLoopsHit = 0 + executionInfo.DailyChildOrgAgentExecutionsSuccessful = 0 + executionInfo.DailyChildOrgAgentExecutionsFailed = 0 + executionInfo.DailyChildOrgAgentCachedTokens = 0 + executionInfo.DailyChildOrgAgentMaxLoopsHit = 0 + + executionInfo.WeeklyAppExecutions = 0 + executionInfo.WeeklyChildAppExecutions = 0 + executionInfo.WeeklyAppExecutionsFailed = 0 + executionInfo.WeeklySubflowExecutions = 0 + executionInfo.WeeklyWorkflowExecutions = 0 + executionInfo.WeeklyWorkflowExecutionsFinished = 0 + executionInfo.WeeklyWorkflowExecutionsFailed = 0 + executionInfo.WeeklyOrgSyncActions = 0 + executionInfo.WeeklyCloudExecutions = 0 + executionInfo.WeeklyOnpremExecutions = 0 + executionInfo.WeeklyChildWorkflowExecutions = 0 + + for additionIndex := range executionInfo.Additions { + executionInfo.Additions[additionIndex].Value = 0 + executionInfo.Additions[additionIndex].DailyValue = 0 + } + } else { + // Update today's stats on each increment + lastIdx := len(executionInfo.DailyStatistics) - 1 + executionInfo.DailyStatistics[lastIdx].AppExecutions = executionInfo.DailyAppExecutions + executionInfo.DailyStatistics[lastIdx].ChildAppExecutions = executionInfo.DailyChildAppExecutions + executionInfo.DailyStatistics[lastIdx].AppExecutionsFailed = executionInfo.DailyAppExecutionsFailed + executionInfo.DailyStatistics[lastIdx].SubflowExecutions = executionInfo.DailySubflowExecutions + executionInfo.DailyStatistics[lastIdx].WorkflowExecutions = executionInfo.DailyWorkflowExecutions + executionInfo.DailyStatistics[lastIdx].WorkflowExecutionsFinished = executionInfo.DailyWorkflowExecutionsFinished + executionInfo.DailyStatistics[lastIdx].WorkflowExecutionsFailed = executionInfo.DailyWorkflowExecutionsFailed + executionInfo.DailyStatistics[lastIdx].OrgSyncActions = executionInfo.DailyOrgSyncActions + executionInfo.DailyStatistics[lastIdx].CloudExecutions = executionInfo.DailyCloudExecutions + executionInfo.DailyStatistics[lastIdx].OnpremExecutions = executionInfo.DailyOnpremExecutions + executionInfo.DailyStatistics[lastIdx].AIUsage = executionInfo.DailyAIUsage + executionInfo.DailyStatistics[lastIdx].ApiUsage = executionInfo.DailyApiUsage + executionInfo.DailyStatistics[lastIdx].Additions = executionInfo.Additions + executionInfo.DailyStatistics[lastIdx].AgentExecutions = executionInfo.DailyAgentExecutions + executionInfo.DailyStatistics[lastIdx].AgentTokens = executionInfo.DailyAgentTokens + executionInfo.DailyStatistics[lastIdx].AgentInputTokens = executionInfo.DailyAgentInputTokens + executionInfo.DailyStatistics[lastIdx].ChildOrgAgentInputTokens = executionInfo.DailyChildOrgAgentInputTokens + executionInfo.DailyStatistics[lastIdx].AgentOutputTokens = executionInfo.DailyAgentOutputTokens + executionInfo.DailyStatistics[lastIdx].ChildOrgAgentOutputTokens = executionInfo.DailyChildOrgAgentOutputTokens + executionInfo.DailyStatistics[lastIdx].ChildOrgAiUsage = executionInfo.DailyChildOrgAiUsage + executionInfo.DailyStatistics[lastIdx].ChildOrgAgentExecutions = executionInfo.DailyChildOrgAgentExecutions + executionInfo.DailyStatistics[lastIdx].ChildOrgAgentTokens = executionInfo.DailyChildOrgAgentTokens + executionInfo.DailyStatistics[lastIdx].DailySMSUsage = executionInfo.DailySMSUsage + executionInfo.DailyStatistics[lastIdx].DailyChildOrgSMSUsage = executionInfo.DailyChildOrgSMSUsage + executionInfo.DailyStatistics[lastIdx].DailyEmailUsage = executionInfo.DailyEmailUsage + executionInfo.DailyStatistics[lastIdx].DailyChildOrgEmailUsage = executionInfo.DailyChildOrgEmailUsage + + executionInfo.DailyStatistics[lastIdx].AgentExecutionsSuccessful = executionInfo.DailyAgentExecutionsSuccessful + executionInfo.DailyStatistics[lastIdx].AgentExecutionsFailed = executionInfo.DailyAgentExecutionsFailed + executionInfo.DailyStatistics[lastIdx].AgentCachedTokens = executionInfo.DailyAgentCachedTokens + executionInfo.DailyStatistics[lastIdx].AgentMaxLoopsHit = executionInfo.DailyAgentMaxLoopsHit + + executionInfo.DailyStatistics[lastIdx].ChildOrgAgentExecutionsSuccessful = executionInfo.DailyChildOrgAgentExecutionsSuccessful + executionInfo.DailyStatistics[lastIdx].ChildOrgAgentExecutionsFailed = executionInfo.DailyChildOrgAgentExecutionsFailed + executionInfo.DailyStatistics[lastIdx].ChildOrgAgentCachedTokens = executionInfo.DailyChildOrgAgentCachedTokens + executionInfo.DailyStatistics[lastIdx].ChildOrgAgentMaxLoopsHit = executionInfo.DailyChildOrgAgentMaxLoopsHit + } now := time.Now() @@ -1443,6 +1799,20 @@ func handleDailyCacheUpdate(executionInfo *ExecutionInfo) *ExecutionInfo { executionInfo.MonthlyAgentInputTokens = 0 executionInfo.MonthlyAgentOutputTokens = 0 executionInfo.MonthlyAgentCachedTokens = 0 + executionInfo.MonthlyChildOrgAiUsage = 0 + executionInfo.MonthlyChildOrgAgentExecutions = 0 + executionInfo.MonthlyChildOrgAgentTokens = 0 + executionInfo.MonthlyChildOrgAgentInputTokens = 0 + executionInfo.MonthlyChildOrgAgentOutputTokens = 0 + executionInfo.MonthlySMSUsage = 0 + executionInfo.MonthlyChildOrgSMSUsage = 0 + executionInfo.MonthlyEmailUsage = 0 + executionInfo.MonthlyChildOrgEmailUsage = 0 + executionInfo.MonthlyAgentMaxLoopsHit = 0 + executionInfo.MonthlyChildOrgAgentExecutionsSuccessful = 0 + executionInfo.MonthlyChildOrgAgentExecutionsFailed = 0 + executionInfo.MonthlyChildOrgAgentCachedTokens = 0 + executionInfo.MonthlyChildOrgAgentMaxLoopsHit = 0 executionInfo.LastMonthlyResetMonth = currentMonth executionInfo.LastUsageAlertThreshold = 0 executionInfo.MonthlyAIUsageAlertSent = false @@ -1500,6 +1870,80 @@ func checkAndSetAlertCache(ctx context.Context, cacheKey string) bool { return true } +func CheckOnpremUsageAlerts(ctx context.Context, org *Org, onpremMonthlyTotal int64) error { + if !isOnpremAlertEligible(org) { + return nil + } + + onpremLimit := org.SyncFeatures.OnpremAppExecutions.Limit + if onpremLimit <= 0 { + return nil + } + + onpremPercentage := float64(onpremMonthlyTotal) / float64(onpremLimit) * 100 + + allAdmins := []string{} + for _, user := range org.Users { + if user.Role == "admin" { + allAdmins = append(allAdmins, user.Username) + } + } + + if !ArrayContains(allAdmins, "chris@shuffler.io") { + allAdmins = append(allAdmins, "chris@shuffler.io") + } + + if !ArrayContains(allAdmins, "jay@shuffler.io") { + allAdmins = append(allAdmins, "jay@shuffler.io") + } + + changed := false + for index, alert := range org.Billing.OnpremAlertThreshold { + if alert.Email_send || onpremPercentage < float64(alert.Percentage) { + continue + } + + cacheKey := generateAlertCacheKey(org.Id, fmt.Sprintf("onprem_%d", alert.Percentage), allAdmins) + if !checkAndSetAlertCache(ctx, cacheKey) { + continue + } + + usagePercentageStr := fmt.Sprintf("%d%% of your on-premise app runs limit", alert.Percentage) + Subject := fmt.Sprintf("[Shuffle]: You've reached %s for your tenant %s", usagePercentageStr, org.Name) + substitutions := map[string]interface{}{ + "app_runs_usage": onpremMonthlyTotal, + "app_runs_limit": onpremLimit, + "subject_string": usagePercentageStr, + "org_name": org.Name, + "org_id": org.Id, + "admin_email": org.Name, + "app_runs_usage_percentage": int64(onpremPercentage), + } + + err := sendMailSendgridV2( + []string{"support@shuffler.io"}, + Subject, + substitutions, + false, + "d-3678d48b2b7144feb4b0b4cff7045016", + allAdmins, + ) + if err != nil { + log.Printf("[ERROR] Failed sending onprem usage alert mail for org %s: %s", org.Id, err) + continue + } + + org.Billing.OnpremAlertThreshold[index].Email_send = true + changed = true + } + + if changed { + return SetOrg(ctx, *org, org.Id) + } + + return nil +} + func HandleIncrement(dataType string, orgStatistics *ExecutionInfo, increment uint) *ExecutionInfo { appendCustom := false @@ -1603,14 +2047,26 @@ func HandleIncrement(dataType string, orgStatistics *ExecutionInfo, increment ui orgStatistics.TotalAgentMaxLoopsHit += int64(increment) orgStatistics.MonthlyAgentMaxLoopsHit += int64(increment) orgStatistics.DailyAgentMaxLoopsHit += int64(increment) + } else if dataType == "child_org_agent_max_loops_hit" { + orgStatistics.TotalChildOrgAgentMaxLoopsHit += int64(increment) + orgStatistics.MonthlyChildOrgAgentMaxLoopsHit += int64(increment) + orgStatistics.DailyChildOrgAgentMaxLoopsHit += int64(increment) } else if dataType == "agent_tokens" { orgStatistics.TotalAgentTokens += int64(increment) orgStatistics.MonthlyAgentTokens += int64(increment) orgStatistics.DailyAgentTokens += int64(increment) + } else if dataType == "childorg_agent_tokens" { + orgStatistics.TotalChildOrgAgentTokens += int64(increment) + orgStatistics.MonthlyChildOrgAgentTokens += int64(increment) + orgStatistics.DailyChildOrgAgentTokens += int64(increment) } else if dataType == "agent_input_tokens" { orgStatistics.TotalAgentInputTokens += int64(increment) orgStatistics.MonthlyAgentInputTokens += int64(increment) orgStatistics.DailyAgentInputTokens += int64(increment) + } else if dataType == "childorg_agent_input_tokens" { + orgStatistics.TotalChildOrgAgentInputTokens += int64(increment) + orgStatistics.MonthlyChildOrgAgentInputTokens += int64(increment) + orgStatistics.DailyChildOrgAgentInputTokens += int64(increment) } else if dataType == "agent_output_tokens" { orgStatistics.TotalAgentOutputTokens += int64(increment) orgStatistics.MonthlyAgentOutputTokens += int64(increment) @@ -1643,7 +2099,35 @@ func HandleIncrement(dataType string, orgStatistics *ExecutionInfo, increment ui orgStatistics.TotalChildOrgAgentOutputTokens += int64(increment) orgStatistics.MonthlyChildOrgAgentOutputTokens += int64(increment) orgStatistics.DailyChildOrgAgentOutputTokens += int64(increment) - } else if dataType == "child_org_agent_cached_tokens" { + } else if dataType == "send_sms" { + orgStatistics.TotalSMSUsage += int64(increment) + orgStatistics.MonthlySMSUsage += int64(increment) + orgStatistics.DailySMSUsage += int64(increment) + } else if dataType == "childorg_send_sms" { + orgStatistics.TotalChildOrgSMSUsage += int64(increment) + orgStatistics.MonthlyChildOrgSMSUsage += int64(increment) + orgStatistics.DailyChildOrgSMSUsage += int64(increment) + } else if dataType == "send_mail" { + orgStatistics.TotalEmailUsage += int64(increment) + orgStatistics.MonthlyEmailUsage += int64(increment) + orgStatistics.DailyEmailUsage += int64(increment) + } else if dataType == "childorg_send_mail" { + orgStatistics.TotalChildOrgEmailUsage += int64(increment) + orgStatistics.MonthlyChildOrgEmailUsage += int64(increment) + orgStatistics.DailyChildOrgEmailUsage += int64(increment) + } else if dataType == "child_org_agent_executions" { + orgStatistics.TotalChildOrgAgentExecutions += int64(increment) + orgStatistics.MonthlyChildOrgAgentExecutions += int64(increment) + orgStatistics.DailyChildOrgAgentExecutions += int64(increment) + } else if dataType == "child_org_agent_executions_successful" { + orgStatistics.TotalChildOrgAgentExecutionsSuccessful += int64(increment) + orgStatistics.MonthlyChildOrgAgentExecutionsSuccessful += int64(increment) + orgStatistics.DailyChildOrgAgentExecutionsSuccessful += int64(increment) + } else if dataType == "child_org_agent_executions_failed" { + orgStatistics.TotalChildOrgAgentExecutionsFailed += int64(increment) + orgStatistics.MonthlyChildOrgAgentExecutionsFailed += int64(increment) + orgStatistics.DailyChildOrgAgentExecutionsFailed += int64(increment) + } else if dataType == "childorg_agent_cached_tokens" { orgStatistics.TotalChildOrgAgentCachedTokens += int64(increment) orgStatistics.MonthlyChildOrgAgentCachedTokens += int64(increment) orgStatistics.DailyChildOrgAgentCachedTokens += int64(increment) @@ -1771,17 +2255,24 @@ func HandleIncrement(dataType string, orgStatistics *ExecutionInfo, increment ui AppRunsPercentage := float64(totalAppExecutions) / float64(org.SyncFeatures.AppExecutions.Limit) * 100 appRunsUsagePercentageStr := fmt.Sprintf("%d%% of your app runs limit", int64(AppRunsPercentage)) Subject := fmt.Sprintf("[Shuffle]: You've reached %s for your tenant %s", appRunsUsagePercentageStr, org.Name) + aiTokensUsage := orgStatistics.MonthlyAgentTokens + orgStatistics.MonthlyChildOrgAgentTokens + aiTokensUsagePercentage := float64(aiTokensUsage) / float64(org.SyncFeatures.AgentTokens.Limit) * 100 + aiTokensLimit := org.SyncFeatures.AgentTokens.Limit + if aiTokensLimit == 0 { + aiTokensLimit = 10000000 + } substitutions := map[string]interface{}{ - "app_runs_usage": totalAppExecutions, - "app_runs_limit": org.SyncFeatures.AppExecutions.Limit, - "subject_string": appRunsUsagePercentageStr, - "ai_tokens_usage": orgStatistics.MonthlyAgentTokens, - "ai_tokens_limit": org.SyncFeatures.AgentTokens.Limit, - "org_name": org.Name, - "org_id": org.Id, - "admin_email": org.Name, - "app_runs_usage_percentage": int64(AppRunsPercentage), + "app_runs_usage": totalAppExecutions, + "app_runs_limit": org.SyncFeatures.AppExecutions.Limit, + "subject_string": appRunsUsagePercentageStr, + "ai_tokens_usage": aiTokensUsage, + "ai_tokens_limit": aiTokensLimit, + "org_name": org.Name, + "org_id": org.Id, + "admin_email": org.Name, + "app_runs_usage_percentage": int64(AppRunsPercentage), + "ai_tokens_usage_percentage": int64(aiTokensUsagePercentage), } err = sendMailSendgridV2( diff --git a/structs.go b/structs.go index fd4a381d..adea257d 100755 --- a/structs.go +++ b/structs.go @@ -163,13 +163,14 @@ type ExecutionRequest struct { } type RetStruct struct { - Success bool `json:"success"` - SyncFeatures SyncFeatures `json:"sync_features"` - SessionKey string `json:"session_key"` - IntervalSeconds int64 `json:"interval_seconds"` - Subscriptions []PaymentSubscription `json:"subscriptions,omitempty"` - Licensed bool `json:"licensed"` - CloudSyncUrl string `json:"cloud_sync_url,omitempty"` + Success bool `json:"success"` + SyncFeatures SyncFeatures `json:"sync_features"` + SessionKey string `json:"session_key"` + IntervalSeconds int64 `json:"interval_seconds"` + Subscriptions []PaymentSubscription `json:"subscriptions,omitempty"` + Licensed bool `json:"licensed"` + CloudSyncUrl string `json:"cloud_sync_url,omitempty"` + AppRunsHardLimit int64 `json:"app_runs_hard_limit"` } type AppMini struct { @@ -397,31 +398,45 @@ type IncrementInCache struct { type DailyStatistics struct { Date time.Time `json:"date" datastore:"date"` - AppExecutions int64 `json:"app_executions" datastore:"app_executions"` - ChildAppExecutions int64 `json:"child_app_executions" datastore:"child_app_executions"` - AppExecutionsFailed int64 `json:"app_executions_failed" datastore:"app_executions_failed"` - SubflowExecutions int64 `json:"subflow_executions" datastore:"subflow_executions"` - WorkflowExecutions int64 `json:"workflow_executions" datastore:"workflow_executions"` - WorkflowExecutionsFinished int64 `json:"workflow_executions_finished" datastore:"workflow_executions_finished"` - WorkflowExecutionsFailed int64 `json:"workflow_executions_failed" datastore:"workflow_executions_failed"` - OrgSyncActions int64 `json:"org_sync_actions" datastore:"org_sync_actions"` - CloudExecutions int64 `json:"cloud_executions" datastore:"cloud_executions"` - OnpremExecutions int64 `json:"onprem_executions" datastore:"onprem_executions"` - AIUsage int64 `json:"ai_executions" datastore:"ai_executions"` - AgentExecutions int64 `json:"agent_executions" datastore:"agent_executions"` - AgentExecutionsSuccessful int64 `json:"agent_executions_successful" datastore:"agent_executions_successful"` - AgentExecutionsFailed int64 `json:"agent_executions_failed" datastore:"agent_executions_failed"` - AgentTokens int64 `json:"agent_tokens" datastore:"agent_tokens"` - AgentInputTokens int64 `json:"agent_input_tokens" datastore:"agent_input_tokens"` - AgentOutputTokens int64 `json:"agent_output_tokens" datastore:"agent_output_tokens"` - AgentCachedTokens int64 `json:"agent_cached_tokens" datastore:"agent_cached_tokens"` - ChildOrgAgentExecutions int64 `json:"child_org_agent_executions" datastore:"child_org_agent_executions"` - ChildOrgAgentExecutionsSuccessful int64 `json:"child_org_agent_executions_successful" datastore:"child_org_agent_executions_successful"` - ChildOrgAgentExecutionsFailed int64 `json:"child_org_agent_executions_failed" datastore:"child_org_agent_executions_failed"` - ChildOrgAgentTokens int64 `json:"child_org_agent_tokens" datastore:"child_org_agent_tokens"` - ChildOrgAgentInputTokens int64 `json:"child_org_agent_input_tokens" datastore:"child_org_agent_input_tokens"` - ChildOrgAgentOutputTokens int64 `json:"child_org_agent_output_tokens" datastore:"child_org_agent_output_tokens"` - ChildOrgAgentCachedTokens int64 `json:"child_org_agent_cached_tokens" datastore:"child_org_agent_cached_tokens"` + AppExecutions int64 `json:"app_executions" datastore:"app_executions"` + ChildAppExecutions int64 `json:"child_app_executions" datastore:"child_app_executions"` + AppExecutionsFailed int64 `json:"app_executions_failed" datastore:"app_executions_failed"` + SubflowExecutions int64 `json:"subflow_executions" datastore:"subflow_executions"` + WorkflowExecutions int64 `json:"workflow_executions" datastore:"workflow_executions"` + WorkflowExecutionsFinished int64 `json:"workflow_executions_finished" datastore:"workflow_executions_finished"` + WorkflowExecutionsFailed int64 `json:"workflow_executions_failed" datastore:"workflow_executions_failed"` + OrgSyncActions int64 `json:"org_sync_actions" datastore:"org_sync_actions"` + CloudExecutions int64 `json:"cloud_executions" datastore:"cloud_executions"` + OnpremExecutions int64 `json:"onprem_executions" datastore:"onprem_executions"` + AIUsage int64 `json:"ai_executions" datastore:"ai_executions"` + AgentExecutions int64 `json:"agent_executions" datastore:"agent_executions"` + AgentExecutionsSuccessful int64 `json:"agent_executions_successful" datastore:"agent_executions_successful"` + AgentExecutionsFailed int64 `json:"agent_executions_failed" datastore:"agent_executions_failed"` + AgentTokens int64 `json:"agent_tokens" datastore:"agent_tokens"` + AgentInputTokens int64 `json:"agent_input_tokens" datastore:"agent_input_tokens"` + AgentOutputTokens int64 `json:"agent_output_tokens" datastore:"agent_output_tokens"` + AgentCachedTokens int64 `json:"agent_cached_tokens" datastore:"agent_cached_tokens"` + DailyChildOrgAiUsage int64 `json:"daily_child_org_ai_usage" datastore:"daily_child_org_ai_usage"` + DailyChildOrgAgentExecutions int64 `json:"daily_child_org_agent_executions" datastore:"daily_child_org_agent_executions"` + DailyChildOrgAgentExecutionsSuccessful int64 `json:"daily_child_org_agent_executions_successful" datastore:"daily_child_org_agent_executions_successful"` + DailyChildOrgAgentExecutionsFailed int64 `json:"daily_child_org_agent_executions_failed" datastore:"daily_child_org_agent_executions_failed"` + DailyChildOrgAgentTokens int64 `json:"daily_child_org_agent_tokens" datastore:"daily_child_org_agent_tokens"` + DailyChildOrgAgentInputTokens int64 `json:"daily_child_org_agent_input_tokens" datastore:"daily_child_org_agent_input_tokens"` + DailyChildOrgAgentOutputTokens int64 `json:"daily_child_org_agent_output_tokens" datastore:"daily_child_org_agent_output_tokens"` + DailySMSUsage int64 `json:"daily_sms_usage" datastore:"daily_sms_usage"` + DailyChildOrgSMSUsage int64 `json:"daily_child_org_sms_usage" datastore:"daily_child_org_sms_usage"` + DailyEmailUsage int64 `json:"daily_email_usage" datastore:"daily_email_usage"` + DailyChildOrgEmailUsage int64 `json:"daily_child_org_email_usage" datastore:"daily_child_org_email_usage"` + ChildOrgAgentExecutions int64 `json:"child_org_agent_executions" datastore:"child_org_agent_executions"` + ChildOrgAgentTokens int64 `json:"child_org_agent_tokens" datastore:"child_org_agent_tokens"` + ChildOrgAgentInputTokens int64 `json:"child_org_agent_input_tokens" datastore:"child_org_agent_input_tokens"` + ChildOrgAgentOutputTokens int64 `json:"child_org_agent_output_tokens" datastore:"child_org_agent_output_tokens"` + ChildOrgAiUsage int64 `json:"child_org_ai_usage" datastore:"child_org_ai_usage"` + AgentMaxLoopsHit int64 `json:"agent_max_loops_hit" datastore:"agent_max_loops_hit"` + ChildOrgAgentMaxLoopsHit int64 `json:"child_org_agent_max_loops_hit" datastore:"child_org_agent_max_loops_hit"` + ChildOrgAgentExecutionsSuccessful int64 `json:"child_org_agent_executions_successful" datastore:"child_org_agent_executions_successful"` + ChildOrgAgentExecutionsFailed int64 `json:"child_org_agent_executions_failed" datastore:"child_org_agent_executions_failed"` + ChildOrgAgentCachedTokens int64 `json:"child_org_agent_cached_tokens" datastore:"child_org_agent_cached_tokens"` ApiUsage int64 `json:"api_usage" datastore:"api_usage"` AppUsage []AppUsage `json:"app_usage" datastore:"app_usage"` @@ -429,6 +444,24 @@ type DailyStatistics struct { Additions []AdditionalUseConfig `json:"additions,omitempty" datastore:"additions"` } +type Tenants struct { + Name string `json:"name" datastore:"name"` + Id string `json:"id" datastore:"id"` + CreatedAt time.Time `json:"created_at" datastore:"created_at"` + DeletedAt time.Time `json:"deleted_at" datastore:"deleted_at"` + Status string `json:"status" datastore:"status"` +} + +type Locations struct { + OrgId string `json:"org_id"` + OrgName string `json:"org_name"` + Name string `json:"name"` + Id string `json:"id"` + CreatedAt string `json:"created_at"` + DeletedAt string `json:"deleted_at"` + Status string `json:"status"` +} + // Used to be related to users, now related to orgs. // Not directly, but being updated by org actions type ExecutionInfo struct { @@ -458,17 +491,22 @@ type ExecutionInfo struct { TotalAgentTokens int64 `json:"total_agent_tokens" datastore:"total_agent_tokens"` TotalAgentInputTokens int64 `json:"total_agent_input_tokens" datastore:"total_agent_input_tokens"` TotalAgentOutputTokens int64 `json:"total_agent_output_tokens" datastore:"total_agent_output_tokens"` - TotalAgentCachedTokens int64 `json:"total_agent_cached_tokens" datastore:"total_agent_cached_tokens"` TotalAgentMaxLoopsHit int64 `json:"total_agent_max_loops_hit" datastore:"total_agent_max_loops_hit"` - TotalChildOrgAgentExecutions int64 `json:"total_child_org_agent_executions" datastore:"total_child_org_agent_executions"` TotalChildOrgAgentExecutionsSuccessful int64 `json:"total_child_org_agent_executions_successful" datastore:"total_child_org_agent_executions_successful"` TotalChildOrgAgentExecutionsFailed int64 `json:"total_child_org_agent_executions_failed" datastore:"total_child_org_agent_executions_failed"` + TotalChildOrgAgentCachedTokens int64 `json:"total_child_org_agent_cached_tokens" datastore:"total_child_org_agent_cached_tokens"` + TotalChildOrgAgentMaxLoopsHit int64 `json:"total_child_org_agent_max_loops_hit" datastore:"total_child_org_agent_max_loops_hit"` + TotalAgentCachedTokens int64 `json:"total_agent_cached_tokens" datastore:"total_agent_cached_tokens"` + TotalChildOrgAiUsage int64 `json:"total_child_org_ai_usage" datastore:"total_child_org_ai_usage"` + TotalChildOrgAgentExecutions int64 `json:"total_child_org_agent_executions" datastore:"total_child_org_agent_executions"` TotalChildOrgAgentTokens int64 `json:"total_child_org_agent_tokens" datastore:"total_child_org_agent_tokens"` TotalChildOrgAgentInputTokens int64 `json:"total_child_org_agent_input_tokens" datastore:"total_child_org_agent_input_tokens"` TotalChildOrgAgentOutputTokens int64 `json:"total_child_org_agent_output_tokens" datastore:"total_child_org_agent_output_tokens"` - TotalChildOrgAgentCachedTokens int64 `json:"total_child_org_agent_cached_tokens" datastore:"total_child_org_agent_cached_tokens"` - TotalChildOrgAgentMaxLoopsHit int64 `json:"total_child_org_agent_max_loops_hit" datastore:"total_child_org_agent_max_loops_hit"` TotalChildWorkflowExecutions int64 `json:"total_child_workflow_executions" datastore:"total_child_workflow_executions"` + TotalSMSUsage int64 `json:"total_sms_usage" datastore:"total_sms_usage"` + TotalChildOrgSMSUsage int64 `json:"total_child_org_sms_usage" datastore:"total_child_org_sms_usage"` + TotalEmailUsage int64 `json:"total_email_usage" datastore:"total_email_usage"` + TotalChildOrgEmailUsage int64 `json:"total_child_org_email_usage" datastore:"total_child_org_email_usage"` MonthlyApiUsage int64 `json:"monthly_api_usage,omitempty" datastore:"monthly_api_usage"` MonthlyChildAppExecutions int64 `json:"monthly_child_app_executions,omitempty" datastore:"monthly_child_app_executions"` @@ -491,6 +529,7 @@ type ExecutionInfo struct { MonthlyAgentOutputTokens int64 `json:"monthly_agent_output_tokens,omitempty" datastore:"monthly_agent_output_tokens"` MonthlyAgentCachedTokens int64 `json:"monthly_agent_cached_tokens,omitempty" datastore:"monthly_agent_cached_tokens"` MonthlyAgentMaxLoopsHit int64 `json:"monthly_agent_max_loops_hit,omitempty" datastore:"monthly_agent_max_loops_hit"` + MonthlyChildOrgAiUsage int64 `json:"monthly_child_org_ai_usage,omitempty" datastore:"monthly_child_org_ai_usage"` MonthlyChildOrgAgentExecutions int64 `json:"monthly_child_org_agent_executions,omitempty" datastore:"monthly_child_org_agent_executions"` MonthlyChildOrgAgentExecutionsSuccessful int64 `json:"monthly_child_org_agent_executions_successful,omitempty" datastore:"monthly_child_org_agent_executions_successful"` MonthlyChildOrgAgentExecutionsFailed int64 `json:"monthly_child_org_agent_executions_failed,omitempty" datastore:"monthly_child_org_agent_executions_failed"` @@ -499,6 +538,10 @@ type ExecutionInfo struct { MonthlyChildOrgAgentOutputTokens int64 `json:"monthly_child_org_agent_output_tokens,omitempty" datastore:"monthly_child_org_agent_output_tokens"` MonthlyChildOrgAgentCachedTokens int64 `json:"monthly_child_org_agent_cached_tokens,omitempty" datastore:"monthly_child_org_agent_cached_tokens"` MonthlyChildOrgAgentMaxLoopsHit int64 `json:"monthly_child_org_agent_max_loops_hit,omitempty" datastore:"monthly_child_org_agent_max_loops_hit"` + MonthlySMSUsage int64 `json:"monthly_sms_usage,omitempty" datastore:"monthly_sms_usage"` + MonthlyChildOrgSMSUsage int64 `json:"monthly_child_org_sms_usage,omitempty" datastore:"monthly_child_org_sms_usage"` + MonthlyEmailUsage int64 `json:"monthly_email_usage,omitempty" datastore:"monthly_email_usage"` + MonthlyChildOrgEmailUsage int64 `json:"monthly_child_org_email_usage,omitempty" datastore:"monthly_child_org_email_usage"` WeeklyAppExecutions int64 `json:"weekly_app_executions,omitempty" datastore:"weekly_app_executions"` WeeklyChildAppExecutions int64 `json:"weekly_child_app_executions,omitempty" datastore:"weekly_child_app_executions"` @@ -529,10 +572,11 @@ type ExecutionInfo struct { DailyAgentExecutionsSuccessful int64 `json:"daily_agent_executions_successful" datastore:"daily_agent_executions_successful"` DailyAgentExecutionsFailed int64 `json:"daily_agent_executions_failed" datastore:"daily_agent_executions_failed"` DailyAgentTokens int64 `json:"daily_agent_tokens" datastore:"daily_agent_tokens"` - DailyAgentInputTokens int64 `json:"daily_agent_input_tokens" datastore:"daily_agent_input_tokens"` - DailyAgentOutputTokens int64 `json:"daily_agent_output_tokens" datastore:"daily_agent_output_tokens"` DailyAgentCachedTokens int64 `json:"daily_agent_cached_tokens" datastore:"daily_agent_cached_tokens"` DailyAgentMaxLoopsHit int64 `json:"daily_agent_max_loops_hit,omitempty" datastore:"daily_agent_max_loops_hit"` + DailyAgentInputTokens int64 `json:"daily_agent_input_tokens" datastore:"daily_agent_input_tokens"` + DailyAgentOutputTokens int64 `json:"daily_agent_output_tokens" datastore:"daily_agent_output_tokens"` + DailyChildOrgAiUsage int64 `json:"daily_child_org_ai_usage" datastore:"daily_child_org_ai_usage"` DailyChildOrgAgentExecutions int64 `json:"daily_child_org_agent_executions" datastore:"daily_child_org_agent_executions"` DailyChildOrgAgentExecutionsSuccessful int64 `json:"daily_child_org_agent_executions_successful" datastore:"daily_child_org_agent_executions_successful"` DailyChildOrgAgentExecutionsFailed int64 `json:"daily_child_org_agent_executions_failed" datastore:"daily_child_org_agent_executions_failed"` @@ -541,6 +585,10 @@ type ExecutionInfo struct { DailyChildOrgAgentOutputTokens int64 `json:"daily_child_org_agent_output_tokens" datastore:"daily_child_org_agent_output_tokens"` DailyChildOrgAgentCachedTokens int64 `json:"daily_child_org_agent_cached_tokens" datastore:"daily_child_org_agent_cached_tokens"` DailyChildOrgAgentMaxLoopsHit int64 `json:"daily_child_org_agent_max_loops_hit,omitempty" datastore:"daily_child_org_agent_max_loops_hit"` + DailySMSUsage int64 `json:"daily_sms_usage" datastore:"daily_sms_usage"` + DailyChildOrgSMSUsage int64 `json:"daily_child_org_sms_usage" datastore:"daily_child_org_sms_usage"` + DailyEmailUsage int64 `json:"daily_email_usage" datastore:"daily_email_usage"` + DailyChildOrgEmailUsage int64 `json:"daily_child_org_email_usage" datastore:"daily_child_org_email_usage"` HourlyAppExecutions int64 `json:"hourly_app_executions,omitempty" datastore:"hourly_app_executions"` HourlyChildAppExecutions int64 `json:"hourly_child_app_executions,omitempty" datastore:"hourly_child_app_executions"` @@ -555,10 +603,17 @@ type ExecutionInfo struct { HourlyOnpremExecutions int64 `json:"hourly_onprem_executions,omitempty" datastore:"hourly_onprem_executions"` HourlyAIUsage int64 `json:"hourly_ai_executions,omitempty" datastore:"hourly_ai_executions"` + AnnualAppExecutions int64 `json:"annual_app_executions,omitempty" datastore:"annual_app_executions"` + AnnualChildAppExecutions int64 `json:"annual_child_app_executions,omitempty" datastore:"annual_child_app_executions"` + // These are just here in case we get use of them TotalApiUsage int64 `json:"total_api_usage" datastore:"total_api_usage"` DailyApiUsage int64 `json:"daily_api_usage" datastore:"daily_api_usage"` + // Store only deleted tenants here. So it doesn't grow out of 1MB datastore limit + Tenants []Tenants `json:"tenants" datastore:"tenants"` + Locations []Locations `json:"locations" datastore:"locations"` + Additions []AdditionalUseConfig `json:"additions,omitempty" datastore:"additions"` LastMonthlyResetMonth int `json:"last_monthly_reset_month" datastore:"last_monthly_reset_month"` LastUsageAlertThreshold int64 `json:"last_usage_alert_threshold" datastore:"last_usage_alert_threshold"` @@ -566,6 +621,11 @@ type ExecutionInfo struct { MonthlyAIUsageAlertSent bool `json:"monthly_ai_usage_alert_sent" datastore:"monthly_ai_usage_alert_sent"` } +type MultiRegionStatsEntry struct { + OrgId string `json:"org_id"` + DailyStatistics []DailyStatistics `json:"daily_statistics"` +} + type AdditionalUseConfig struct { Key string `json:"key" datastore:"key"` Value int64 `json:"value" datastore:"value"` @@ -1047,6 +1107,16 @@ type LeadInfo struct { ChannelPartner bool `json:"channel_partner,omitempty" datastore:"channel_partner"` Creator bool `json:"creator,omitempty" datastore:"creator"` + + ShuffleEnterpriseLicenseOldCustomer bool `json:"shuffle_enterprise_license_old_customer,omitempty" datastore:"shuffle_enterprise_license_old_customer"` + ScaleLicenseCloudTrial bool `json:"scale_license_cloud_trial,omitempty" datastore:"scale_license_cloud_trial"` + OpenSourceLicense bool `json:"opensource_license,omitempty" datastore:"opensource_license"` + ScaleLicenseCloudCustomer bool `json:"scale_license_cloud_customer,omitempty" datastore:"scale_license_cloud_customer"` + ScaleLicenseOnpremCustomer bool `json:"scale_license_onprem_customer,omitempty" datastore:"scale_license_onprem_customer"` + BusinessLicenseCloud bool `json:"business_license_cloud,omitempty" datastore:"business_license_cloud"` + BusinessLicenseOnprem bool `json:"business_license_onprem,omitempty" datastore:"business_license_onprem"` + EnterpriseLicenseCloud bool `json:"enterprise_license_cloud,omitempty" datastore:"enterprise_license_cloud"` + EnterpriseLicenseOnprem bool `json:"enterprise_license_onprem,omitempty" datastore:"enterprise_license_onprem"` } // Partners Structs @@ -1117,6 +1187,8 @@ type OnpremLicense struct { AppRuns OnpremLimits `json:"app_runs" datastore:"app_runs"` Timeout string `json:"timeout" datastore:"timeout"` Branding bool `json:"branding" datastore:"branding"` + AppRunsGrouping bool `json:"app_runs_grouping" datastore:"app_runs_grouping"` + StartDate string `json:"start_date" datastore:"start_date"` } type Org struct { @@ -1173,11 +1245,14 @@ type Org struct { } type Billing struct { - Email string `json:"Email" datastore:"Email"` - AppRunsHardLimit int64 `json:"app_runs_hard_limit" datastore:"app_runs_hard_limit"` - AlertThreshold []AlertThreshold `json:"AlertThreshold" datastore:"AlertThreshold"` - Consultation Consultation `json:"Consultation" datastore:"Consultation"` - InternalAppRunsHardLimit int64 `json:"internal_app_runs_hard_limit" datastore:"internal_app_runs_hard_limit"` + Email string `json:"Email" datastore:"Email"` + AppRunsHardLimit int64 `json:"app_runs_hard_limit" datastore:"app_runs_hard_limit"` + AlertThreshold []AlertThreshold `json:"AlertThreshold" datastore:"AlertThreshold"` + OnpremAlertThreshold []AlertThreshold `json:"OnpremAlertThreshold" datastore:"OnpremAlertThreshold"` + Consultation Consultation `json:"Consultation" datastore:"Consultation"` + InternalAppRunsHardLimit int64 `json:"internal_app_runs_hard_limit" datastore:"internal_app_runs_hard_limit"` + DefaultAlertsApplied bool `json:"default_alerts_applied" datastore:"default_alerts_applied"` + DefaultOnpremAlertsApplied bool `json:"default_onprem_alerts_applied" datastore:"default_onprem_alerts_applied"` } type AlertThreshold struct { @@ -1423,32 +1498,33 @@ type MailLevel struct { } type SyncFeatures struct { - Editing bool `json:"editing" datastore:"editing"` - MailSent []MailLevel `json:"mail_sent" datastore:"mail_sent"` - AppExecutions SyncData `json:"app_executions" datastore:"app_executions"` - OnpremAppExecutions SyncData `json:"onprem_app_executions" datastore:"onprem_app_executions"` - MultiEnv SyncData `json:"multi_env" datastore:"multi_env"` - MultiTenant SyncData `json:"multi_tenant" datastore:"multi_tenant"` - MultiRegion SyncData `json:"multi_region" datastore:"multi_region"` - Webhook SyncData `json:"webhook" datastore:"webhook"` - Schedules SyncData `json:"schedules" datastore:"schedules"` - UserInput SyncData `json:"user_input" datastore:"user_input"` - SendMail SyncData `json:"send_mail" datastore:"send_mail"` - SendSms SyncData `json:"send_sms" datastore:"send_sms"` - Updates SyncData `json:"updates" datastore:"updates"` - EmailTrigger SyncData `json:"email_trigger" datastore:"email_trigger"` - Notifications SyncData `json:"notifications" datastore:"notifications"` - Workflows SyncData `json:"workflows" datastore:"workflows"` - Autocomplete SyncData `json:"autocomplete" datastore:"autocomplete"` - WorkflowExecutions SyncData `json:"workflow_executions" datastore:"workflow_executions"` - Authentication SyncData `json:"authentication" datastore:"authentication"` - Schedule SyncData `json:"schedule" datastore:"schedule"` - Apps SyncData `json:"apps" datastore:"apps"` - ShuffleGPT SyncData `json:"shuffle_gpt" datastore:"shuffle_gpt"` - Branding SyncData `json:"branding" datastore:"branding"` - AgentExecutions SyncData `json:"agent_executions" datastore:"agent_executions"` - AgentTokens SyncData `json:"agent_tokens" datastore:"agent_tokens"` - Multiplayer SyncData `json:"multiplayer" datastore:"multiplayer"` + Editing bool `json:"editing" datastore:"editing"` + MailSent []MailLevel `json:"mail_sent" datastore:"mail_sent"` + AppExecutions SyncData `json:"app_executions" datastore:"app_executions"` + OnpremAppExecutions SyncData `json:"onprem_app_executions" datastore:"onprem_app_executions"` + AnnualAppRunsGrouping SyncData `json:"annual_app_runs_grouping" datastore:"annual_app_runs_grouping"` + MultiEnv SyncData `json:"multi_env" datastore:"multi_env"` + MultiTenant SyncData `json:"multi_tenant" datastore:"multi_tenant"` + MultiRegion SyncData `json:"multi_region" datastore:"multi_region"` + Webhook SyncData `json:"webhook" datastore:"webhook"` + Schedules SyncData `json:"schedules" datastore:"schedules"` + UserInput SyncData `json:"user_input" datastore:"user_input"` + SendMail SyncData `json:"send_mail" datastore:"send_mail"` + SendSms SyncData `json:"send_sms" datastore:"send_sms"` + Updates SyncData `json:"updates" datastore:"updates"` + EmailTrigger SyncData `json:"email_trigger" datastore:"email_trigger"` + Notifications SyncData `json:"notifications" datastore:"notifications"` + Workflows SyncData `json:"workflows" datastore:"workflows"` + Autocomplete SyncData `json:"autocomplete" datastore:"autocomplete"` + WorkflowExecutions SyncData `json:"workflow_executions" datastore:"workflow_executions"` + Authentication SyncData `json:"authentication" datastore:"authentication"` + Schedule SyncData `json:"schedule" datastore:"schedule"` + Apps SyncData `json:"apps" datastore:"apps"` + ShuffleGPT SyncData `json:"shuffle_gpt" datastore:"shuffle_gpt"` + Branding SyncData `json:"branding" datastore:"branding"` + AgentExecutions SyncData `json:"agent_executions" datastore:"agent_executions"` + AgentTokens SyncData `json:"agent_tokens" datastore:"agent_tokens"` + Multiplayer SyncData `json:"multiplayer" datastore:"multiplayer"` } type SyncData struct {