[board] Record views and feedback results without the events consent

CHANGELOG.md

@@ -24,6 +24,7 @@
 * ! Minor breaking change ! The feature interfaces of "Countly.instance()" ("events()", "views()", "crashes()", "location()", "userProfile()", "remoteConfig()", "feedback()", "content()" and "deviceId()") no longer return null.
   Before initialization, while the consent of a feature is not given, or when a feature is not enabled, they return an interface that ignores calls until the feature runs and passes an error to the callbacks of the calls it ignores.
   An interface you keep sends its calls to the feature once it runs. Use "Countly.isTracking" to check whether a feature runs.
+* ! Minor breaking change ! Views and feedback widget results are now recorded without "Config.Feature.Events" and its consent: their own feature and its consent are enough. Before, they were dropped while the events feature did not run. Removing the events consent still drops the custom and timed events not sent yet, and now keeps the views and feedback widget results whose own consent is still given.
 
 * Added support for SDK behavior settings that let the server control the SDK:
   * Switches for tracking, networking, sessions, views, custom events, crash reporting, automatic crash reporting, location, and console logging

sdk-java/src/main/java/ly/count/sdk/java/internal/EventQueue.java

@@ -2,7 +2,9 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Iterator;
 import java.util.List;
+import java.util.function.Predicate;
 import javax.annotation.Nonnull;
 
 public class EventQueue {
@@ -108,4 +110,40 @@ public void clear() {
             eventQueueMemoryCache.clear();
         }
     }
+
+    /**
+     * Removes the events that match and stores the ones left.
+     *
+     * @param filter which events to remove
+     * @return the removed events, in queue order
+     */
+    @Nonnull List<EventImpl> take(@Nonnull Predicate<EventImpl> filter) {
+        List<EventImpl> taken = new ArrayList<>();
+        synchronized (lockEQ) {
+            Iterator<EventImpl> events = eventQueueMemoryCache.iterator();
+            while (events.hasNext()) {
+                EventImpl event = events.next();
+                if (filter.test(event)) {
+                    taken.add(event);
+                    events.remove();
+                }
+            }
+            if (!taken.isEmpty()) {
+                SDKCore.instance.sdkStorage.storeEventQueue(joinEvents(eventQueueMemoryCache));
+            }
+        }
+        return taken;
+    }
+
+    /**
+     * Counts the queued events that match.
+     *
+     * @param filter which events to count
+     * @return the number of matching events
+     */
+    int count(@Nonnull Predicate<EventImpl> filter) {
+        synchronized (lockEQ) {
+            return (int) eventQueueMemoryCache.stream().filter(filter).count();
+        }
+    }
 }

sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleBase.java

@@ -52,21 +52,35 @@ public void init(InternalConfig config) {
     public void stop(InternalConfig config, boolean clear) {
     }
 
+    /**
+     * Whether the feature of this module runs while the SDK runs the module. The SDK runs a module
+     * only for its feature, so this is {@code true}, except for a module it also runs for other
+     * features.
+     *
+     * @return {@code true} while the feature of this module runs
+     */
+    boolean featureRuns() {
+        return true;
+    }
+
     /**
      * The instance of this module that the SDK runs, which every call through an interface of this
      * module goes to: this one, or the running one when the interface was handed out before that
-     * started or kept from an instance that stopped. A call while none runs, as before init or while
-     * the consent of the feature is not given, is ignored and logged.
+     * started or kept from an instance that stopped. A call while none runs or its feature does not,
+     * as before init or while the consent of the feature is not given, is ignored and logged.
      *
      * @param call the call, starting with the bracketed name of the interface, for the log line
      * @param <M> the module type
-     * @return the running instance, {@code null} while none runs
+     * @return the running instance, {@code null} while none runs or its feature does not
      */
     @SuppressWarnings("unchecked")
     @Nullable
     <M extends ModuleBase> M runningInstance(@Nonnull String call) {
         SDKCore core = SDKCore.instance;
         ModuleBase running = core == null ? null : core.module(getClass());
+        if (running != null && !running.featureRuns()) {
+            running = null;
+        }
         if (running == null && core != null && core.L != null) {
             core.L.w(call + ", the feature does not run, as its consent is not given or it is not enabled, ignoring the call");
         }

sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleEvents.java

@@ -5,6 +5,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
@@ -17,8 +18,8 @@
 public class ModuleEvents extends ModuleBase {
     /**
      * The keys of the events the SDK records itself, which the custom event settings of the SDK
-     * behavior settings do not apply to. Any other key is a custom event, one starting with
-     * {@code [CLY]_} too.
+     * behavior settings do not apply to, and which the views and feedback features record without
+     * the events consent. Any other key is a custom event, one starting with {@code [CLY]_} too.
      */
     static final Set<String> INTERNAL_EVENT_KEYS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
         ModuleViews.KEY_VIEW_EVENT, FeedbackWidgetType.nps.eventKey, FeedbackWidgetType.survey.eventKey, FeedbackWidgetType.rating.eventKey)));
@@ -148,7 +149,56 @@ public void stop(InternalConfig config, final boolean clear) {
     }
 
     /**
-     * Moves every queued event into one request.
+     * The events feature runs while it is enabled and, when consent is required, its consent is
+     * given. This module also runs without it, for the events of the views and feedback features.
+     *
+     * @return {@code true} while custom events can be recorded and sent
+     */
+    @Override
+    boolean featureRuns() {
+        return SDKCore.enabled(CoreFeature.Events);
+    }
+
+    /**
+     * Drops the timed events and the queued events whose feature no longer runs, as the events
+     * consent was removed: the custom events, and the views and feedback widget results whose own
+     * consent is not given either. The next custom event starts a new chain of previous event IDs.
+     */
+    synchronized void dropEventsWithoutConsent() {
+        List<EventImpl> dropped = eventQueue.take(event -> !SDKCore.enabled(featureOf(event.key)));
+        L.d("[ModuleEvents] dropEventsWithoutConsent, the events consent was removed, dropped [" + dropped.size() + "] queued events and [" + timedEvents.size() + "] timed events");
+        timedEvents.clear();
+        previousEventId = null;
+    }
+
+    /**
+     * Whether a key is the key of a custom event rather than of one the SDK records for the views or
+     * feedback feature, as only custom events need the events consent.
+     *
+     * @param key the key of the event
+     * @return {@code true} for a custom event
+     */
+    private static boolean isCustomEventKey(@Nonnull String key) {
+        return !INTERNAL_EVENT_KEYS.contains(key);
+    }
+
+    /**
+     * The feature whose consent an event needs: the views or the feedback feature for the events the
+     * SDK records for them, the events feature for custom events.
+     *
+     * @param key the key of the event
+     * @return the feature
+     */
+    private static CoreFeature featureOf(@Nonnull String key) {
+        if (ModuleViews.KEY_VIEW_EVENT.equals(key)) {
+            return CoreFeature.Views;
+        }
+        return isCustomEventKey(key) ? CoreFeature.Events : CoreFeature.Feedback;
+    }
+
+    /**
+     * Moves the queued events into one request: all of them while the events feature runs, otherwise
+     * the ones of the views and feedback features.
      *
      * @param deviceId the device ID the events belong to, {@code null} for the current one
      */
@@ -157,7 +207,8 @@ private void addEventsToRequestQ(@Nullable String deviceId) {
     }
 
     /**
-     * Moves every queued event into one request.
+     * Moves the queued events into one request: all of them while the events feature runs, otherwise
+     * the ones of the views and feedback features.
      *
      * @param deviceId the device ID the events belong to, {@code null} for the current one
      * @param journeyTrigger whether the events include a journey trigger, whose request refreshes
@@ -166,19 +217,21 @@ private void addEventsToRequestQ(@Nullable String deviceId) {
     private synchronized void addEventsToRequestQ(@Nullable String deviceId, boolean journeyTrigger) {
         L.d("[ModuleEvents] addEventsToRequestQ");
 
-        if (eventQueue.getEQ().isEmpty()) {
-            L.d("[ModuleEvents] addEventsToRequestQ, eventQueueMemoryCache is empty, skipping");
+        boolean allEvents = featureRuns();
+        //custom events restored from an earlier run wait until the events feature runs, the others never need it
+        List<EventImpl> events = eventQueue.take(event -> allEvents || !isCustomEventKey(event.key));
+        if (events.isEmpty()) {
+            L.d("[ModuleEvents] addEventsToRequestQ, no events to send, skipping");
             return;
         }
 
         Request request = new Request();
         if (deviceId != null) {
             request.params.add("device_id", deviceId);
         }
-        request.params.arr("events").put(eventQueue.getEQ()).add();
+        request.params.arr("events").put(events).add();
         request.own(ModuleEvents.class);
 
-        eventQueue.clear();
         if (!journeyTrigger) {
             ModuleRequests.pushAsync(internalConfig, request);
             return;
@@ -211,8 +264,13 @@ protected void recordEventInternal(String key, int count, Double sum, Double dur
             return;
         }
 
-        boolean isCustomEvent = !INTERNAL_EVENT_KEYS.contains(key);
+        boolean isCustomEvent = isCustomEventKey(key);
         if (isCustomEvent) {
+            if (!featureRuns()) {
+                L.d("[ModuleEvents] recordEventInternal, the events feature does not run, as its consent is not given or it is not enabled, event [" + key + "] will not be recorded");
+                return;
+            }
+
             if (!configProvider.getCustomEventTrackingEnabled()) {
                 L.d("[ModuleEvents] recordEventInternal, custom event tracking is disabled by the SDK behavior settings, event [" + key + "] will not be recorded");
                 return;
@@ -328,7 +386,8 @@ void checkEventQueueToSend(boolean forceSend) {
     }
 
     /**
-     * Sends the queued events when asked to or when the queue reached its threshold.
+     * Sends the queued events when asked to or when the ones the next request carries reached the
+     * threshold, so custom events waiting for the events feature never make the others go one by one.
      *
      * @param forceSend whether to send whatever is queued
      * @param journeyTrigger whether the queue holds a journey trigger
@@ -338,8 +397,11 @@ private void checkEventQueueToSend(boolean forceSend, boolean journeyTrigger) {
             L.d("[ModuleEvents] checkEventQueueToSend, the events of the previous device ID are being ended, they are sent together under that ID");
             return;
         }
-        L.d("[ModuleEvents] queue size:[" + eventQueue.eqSize() + "] || forceSend: " + forceSend);
-        if (forceSend || eventQueue.eqSize() >= internalConfig.getConfigurationProvider().getEventQueueSizeThreshold()) {
+        int queued = eventQueue.eqSize();
+        L.d("[ModuleEvents] queue size:[" + queued + "] || forceSend: " + forceSend);
+        int threshold = internalConfig.getConfigurationProvider().getEventQueueSizeThreshold();
+        boolean thresholdReached = queued >= threshold && (featureRuns() || eventQueue.count(event -> !isCustomEventKey(event.key)) >= threshold);
+        if (forceSend || thresholdReached) {
             addEventsToRequestQ(null, journeyTrigger);
         }
     }

sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleFeedback.java

@@ -267,7 +267,12 @@ private void reportFeedbackWidgetManuallyInternal(CountlyFeedbackWidget widgetIn
             segm.putAll(UtilsInternalLimits.truncateStringValues(widgetResult, maxValueSize, L, "[ModuleFeedback] reportFeedbackWidgetManuallyInternal"));
         }
 
-        Countly.instance().events().recordEvent(widgetInfo.type.eventKey, segm);
+        ModuleEvents events = internalConfig.sdk.module(ModuleEvents.class);
+        if (events == null) {
+            L.e("[ModuleFeedback] reportFeedbackWidgetManuallyInternal, events module is not initialized");
+            return;
+        }
+        events.recordEventInternal(widgetInfo.type.eventKey, 1, null, null, segm, null);
     }
 
     private void putIfNotNull(Map<String, Object> segmentation, String key, Object value) {

sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleUserProfile.java

@@ -454,7 +454,7 @@ protected void saveInternal() {
             return;
         }
 
-        ModuleEvents events = internalConfig.sdk.runningModule(CoreFeature.Events, ModuleEvents.class);
+        ModuleEvents events = internalConfig.sdk.module(ModuleEvents.class);
         if (internalConfig.isAutoSendUserProperties() && events != null) {
             events.checkEventQueueToSend(true);
         }

sdk-java/src/main/java/ly/count/sdk/java/internal/SDKCore.java

@@ -142,7 +142,8 @@ protected static void registerDefaultModuleMappings() {
      * @return {@code true} if consent has been given
      */
     public boolean isTracking(Integer feat) {
-        return modules.containsKey(feat);
+        ModuleBase module = modules.get(feat);
+        return module != null && module.featureRuns();
     }
 
     private void onTimer() {
@@ -285,11 +286,20 @@ public void onConsentRemoval(final InternalConfig config, int noConsent) {
             }
         }
 
+        boolean removingEventsConsent = removingConsent(noConsent, CoreFeature.Events);
         consents = consents & ~noConsent;
         L.dropLinesWithoutConsent();
 
+        if (removingEventsConsent) {
+            ModuleEvents events = module(ModuleEvents.class);
+            if (events != null) {
+                events.dropEventsWithoutConsent();
+            }
+        }
+
         for (CoreFeature feature : moduleMappings.keySet()) {
-            if (feature == CoreFeature.Sessions || (noConsent & feature.getIndex()) == 0) {
+            //the events module keeps running, for the events of the views and feedback features
+            if (feature == CoreFeature.Sessions || feature == CoreFeature.Events || (noConsent & feature.getIndex()) == 0) {
                 continue;
             }
             ModuleBase existing = module(moduleMappings.get(feature));
@@ -345,6 +355,8 @@ protected void buildModules(InternalConfig config, int features) throws IllegalA
             modules.put(-1, moduleConfiguration);
         }
         modules.put(0, new ModuleConnectionTest());
+        //the events module queues the events of the views and feedback features too, which need only their own consent
+        modules.put(CoreFeature.Events.getIndex(), new ModuleEvents());
         modules.put(CoreFeature.Sessions.getIndex(), new ModuleSessions());
         modules.put(CoreFeature.UserProfiles.getIndex(), new ModuleUserProfile());
 
@@ -850,8 +862,10 @@ public static File sdkStorageDirectory() {
     }
 
     public static boolean enabled(int feature) {
-        return (feature & instance.consents) == feature &&
-            (feature & instance.config().getFeatures1()) == feature;
+        SDKCore core = instance;
+        InternalConfig config = core == null ? null : core.config;
+        return config != null && (feature & core.consents) == feature &&
+            (feature & config.getFeatures1()) == feature;
     }
 
     public static boolean enabled(CoreFeature feature) {
@@ -1089,7 +1103,8 @@ private <M extends ModuleBase, I> I runningInterface(@Nullable CoreFeature conse
     }
 
     /**
-     * The module of a feature while it runs: it was built and the feature's consent is given.
+     * The module of a feature while it runs: it was built, the feature is enabled and, when consent
+     * is required, the feature's consent is given.
      *
      * @param feature the feature
      * @param moduleClass the class of its module
@@ -1099,6 +1114,6 @@ private <M extends ModuleBase, I> I runningInterface(@Nullable CoreFeature conse
     @Nullable
     <M extends ModuleBase> M runningModule(@Nonnull CoreFeature feature, @Nonnull Class<M> moduleClass) {
         M module = module(moduleClass);
-        return module != null && hasConsentForFeature(feature) ? module : null;
+        return module != null && enabled(feature) ? module : null;
     }
 }

sdk-java/src/main/java/ly/count/sdk/java/internal/TimedEvents.java

@@ -125,7 +125,7 @@ public boolean restore(byte[] data, Log L) {
             while (l-- > 0) {
                 String key = stream.readUTF();
                 EventImpl event = EventImpl.fromJSON(stream.readUTF(), this, L);
-                ModuleEvents eventsModule = SDKCore.instance.module(ModuleEvents.class);
+                ModuleEvents eventsModule = SDKCore.instance.runningModule(CoreFeature.Events, ModuleEvents.class);
                 if (eventsModule != null) {
                     eventsModule.timedEvents.put(key, event);
                 } else {

sdk-java/src/test/java/ly/count/sdk/java/internal/CountlyFacadeTests.java

@@ -324,7 +324,8 @@ public void consentGating_handsOutInterfacesThatIgnoreCalls_untilTheConsentIsGiv
      * Verifies that an events interface handed out before init and one handed out before the consent
      * record once the consent is given, that a views interface handed out before the consent starts a
      * view then, and that a running events interface kept across a removal of its consent ignores
-     * calls until the consent is given again and then records into the new module.
+     * calls until the consent is given again and then records, while the removal dropped the custom
+     * events queued and kept the view, which needs only the views consent.
      */
     @Test
     public void aKeptInterface_reachesTheModuleOnceItsFeatureRuns() {
@@ -349,11 +350,11 @@ public void aKeptInterface_reachesTheModuleOnceItsFeatureRuns() {
         ModuleEvents.Events running = Countly.instance().events();
         Countly.onConsentRemoval(Config.Feature.Events);
         running.recordEvent("ignored_after_removal");
-        TestUtils.validateEQSize(0);
+        Assert.assertEquals(Collections.singletonList(ModuleViews.KEY_VIEW_EVENT), TestUtils.keysInEQ());
 
         Countly.onConsent(Config.Feature.Events);
         running.recordEvent("after_consent_again");
-        Assert.assertEquals(Collections.singletonList("after_consent_again"), TestUtils.keysInEQ());
+        Assert.assertEquals(Arrays.asList(ModuleViews.KEY_VIEW_EVENT, "after_consent_again"), TestUtils.keysInEQ());
     }
 
     /**
@@ -453,7 +454,7 @@ public void deprecatedCallsAndTheUserEditor_logNoUserData_beforeTheirConsentChec
      * Verifies that after the location consent is removed the location interface ignores calls while
      * the device ID, events, views and user profile interfaces keep working and recording, and that
      * an events interface kept from before the events consent is removed ignores calls instead of
-     * recording into the stopped module.
+     * recording.
      */
     @Test
     public void consentRemoval_stopsOnlyTheFeaturesItNames_andAKeptInterfaceIgnoresCalls() {