Merge pull request #7909 from TheThingsNetwork/feature/end-device-count

Author: happyRip

CHANGELOG.md

@@ -11,6 +11,7 @@ For details about compatibility between different releases, see the **Commitment
 
 ### Added
 
+- `Count` RPC on `EndDeviceRegistry` to efficiently retrieve the number of end devices in an application.
 - Add tracing for LBS LNS and TTIGW protocol handlers.
 - TTGC LBS Root CUPS claiming support.
 - Configurable Identity Server user login session TTL via `is.user-login.session-ttl`:

api/ttn/lorawan/v3/api.md

@@ -249,6 +249,8 @@
   - [Message `BatchUpdateEndDeviceLastSeenRequest.EndDeviceLastSeenUpdate`](#ttn.lorawan.v3.BatchUpdateEndDeviceLastSeenRequest.EndDeviceLastSeenUpdate)
   - [Message `BoolValue`](#ttn.lorawan.v3.BoolValue)
   - [Message `ConvertEndDeviceTemplateRequest`](#ttn.lorawan.v3.ConvertEndDeviceTemplateRequest)
+  - [Message `CountEndDevicesRequest`](#ttn.lorawan.v3.CountEndDevicesRequest)
+  - [Message `CountEndDevicesResponse`](#ttn.lorawan.v3.CountEndDevicesResponse)
   - [Message `CreateEndDeviceRequest`](#ttn.lorawan.v3.CreateEndDeviceRequest)
   - [Message `DevAddrPrefix`](#ttn.lorawan.v3.DevAddrPrefix)
   - [Message `EndDevice`](#ttn.lorawan.v3.EndDevice)
@@ -3984,6 +3986,25 @@ Configuration options for static ADR.
 | ----- | ----------- |
 | `format_id` | <p>`string.max_len`: `36`</p><p>`string.pattern`: `^[a-z0-9](?:[-]?[a-z0-9]){2,}$`</p> |
 
+### <a name="ttn.lorawan.v3.CountEndDevicesRequest">Message `CountEndDevicesRequest`</a>
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| `application_ids` | [`ApplicationIdentifiers`](#ttn.lorawan.v3.ApplicationIdentifiers) |  |  |
+| `filters` | [`ListEndDevicesRequest.Filter`](#ttn.lorawan.v3.ListEndDevicesRequest.Filter) | repeated |  |
+
+#### Field Rules
+
+| Field | Validations |
+| ----- | ----------- |
+| `application_ids` | <p>`message.required`: `true`</p> |
+
+### <a name="ttn.lorawan.v3.CountEndDevicesResponse">Message `CountEndDevicesResponse`</a>
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| `count` | [`uint64`](#uint64) |  |  |
+
 ### <a name="ttn.lorawan.v3.CreateEndDeviceRequest">Message `CreateEndDeviceRequest`</a>
 
 | Field | Type | Label | Description |
@@ -4854,6 +4875,7 @@ NsEndDeviceRegistry, the AsEndDeviceRegistry and the JsEndDeviceRegistry.
 | `Get` | [`GetEndDeviceRequest`](#ttn.lorawan.v3.GetEndDeviceRequest) | [`EndDevice`](#ttn.lorawan.v3.EndDevice) | Get the end device with the given identifiers, selecting the fields specified in the field mask. More or less fields may be returned, depending on the rights of the caller. |
 | `GetIdentifiersForEUIs` | [`GetEndDeviceIdentifiersForEUIsRequest`](#ttn.lorawan.v3.GetEndDeviceIdentifiersForEUIsRequest) | [`EndDeviceIdentifiers`](#ttn.lorawan.v3.EndDeviceIdentifiers) | Get the identifiers of the end device that has the given EUIs registered. |
 | `List` | [`ListEndDevicesRequest`](#ttn.lorawan.v3.ListEndDevicesRequest) | [`EndDevices`](#ttn.lorawan.v3.EndDevices) | List end devices in the given application. Similar to Get, this selects the fields given by the field mask. More or less fields may be returned, depending on the rights of the caller. |
+| `Count` | [`CountEndDevicesRequest`](#ttn.lorawan.v3.CountEndDevicesRequest) | [`CountEndDevicesResponse`](#ttn.lorawan.v3.CountEndDevicesResponse) | Count end devices in the given application. |
 | `Update` | [`UpdateEndDeviceRequest`](#ttn.lorawan.v3.UpdateEndDeviceRequest) | [`EndDevice`](#ttn.lorawan.v3.EndDevice) | Update the end device, changing the fields specified by the field mask to the provided values. |
 | `BatchUpdateLastSeen` | [`BatchUpdateEndDeviceLastSeenRequest`](#ttn.lorawan.v3.BatchUpdateEndDeviceLastSeenRequest) | [`.google.protobuf.Empty`](#google.protobuf.Empty) | Update the last seen timestamp for a batch of end devices. |
 | `Delete` | [`EndDeviceIdentifiers`](#ttn.lorawan.v3.EndDeviceIdentifiers) | [`.google.protobuf.Empty`](#google.protobuf.Empty) | Delete the end device with the given IDs. Before deleting an end device it first needs to be deleted from the NsEndDeviceRegistry, the AsEndDeviceRegistry and the JsEndDeviceRegistry. In addition, if the device claimed on a Join Server, it also needs to be unclaimed via the DeviceClaimingServer so it can be claimed in the future. This is NOT done automatically. |
@@ -4866,6 +4888,7 @@ NsEndDeviceRegistry, the AsEndDeviceRegistry and the JsEndDeviceRegistry.
 | `Get` | `GET` | `/api/v3/applications/{end_device_ids.application_ids.application_id}/devices/{end_device_ids.device_id}` |  |
 | `List` | `GET` | `/api/v3/applications/{application_ids.application_id}/devices` |  |
 | `List` | `POST` | `/api/v3/applications/{application_ids.application_id}/devices/filter` | `*` |
+| `Count` | `GET` | `/api/v3/applications/{application_ids.application_id}/devices/count` |  |
 | `Update` | `PUT` | `/api/v3/applications/{end_device.ids.application_ids.application_id}/devices/{end_device.ids.device_id}` | `*` |
 | `Delete` | `DELETE` | `/api/v3/applications/{application_ids.application_id}/devices/{device_id}` |  |
 

api/ttn/lorawan/v3/api.swagger.json

@@ -1185,6 +1185,37 @@
         ]
       }
     },
+    "/applications/{application_ids.application_id}/devices/count": {
+      "get": {
+        "summary": "Count end devices in the given application.",
+        "operationId": "EndDeviceRegistry_Count",
+        "responses": {
+          "200": {
+            "description": "A successful response.",
+            "schema": {
+              "$ref": "#/definitions/v3CountEndDevicesResponse"
+            }
+          },
+          "default": {
+            "description": "An unexpected error response.",
+            "schema": {
+              "$ref": "#/definitions/googlerpcStatus"
+            }
+          }
+        },
+        "parameters": [
+          {
+            "name": "application_ids.application_id",
+            "in": "path",
+            "required": true,
+            "type": "string"
+          }
+        ],
+        "tags": [
+          "EndDeviceRegistry"
+        ]
+      }
+    },
     "/applications/{application_ids.application_id}/devices/filter": {
       "post": {
         "summary": "List end devices in the given application.\nSimilar to Get, this selects the fields given by the field mask.\nMore or less fields may be returned, depending on the rights of the caller.",
@@ -22870,6 +22901,15 @@
         }
       }
     },
+    "v3CountEndDevicesResponse": {
+      "type": "object",
+      "properties": {
+        "count": {
+          "type": "string",
+          "format": "uint64"
+        }
+      }
+    },
     "v3CreateLoginTokenResponse": {
       "type": "object",
       "properties": {

api/ttn/lorawan/v3/end_device.proto

@@ -1346,6 +1346,15 @@ message ListEndDevicesRequest {
   repeated Filter filters = 6;
 }
 
+message CountEndDevicesRequest {
+  ApplicationIdentifiers application_ids = 1 [(validate.rules).message.required = true];
+  repeated ListEndDevicesRequest.Filter filters = 2;
+}
+
+message CountEndDevicesResponse {
+  uint64 count = 1;
+}
+
 message SetEndDeviceRequest {
   EndDevice end_device = 1 [(validate.rules).message.required = true];
   // The names of the end device fields that should be updated.

api/ttn/lorawan/v3/end_device_services.proto

@@ -72,6 +72,11 @@ service EndDeviceRegistry {
     };
   }
 
+  // Count end devices in the given application.
+  rpc Count(CountEndDevicesRequest) returns (CountEndDevicesResponse) {
+    option (google.api.http) = {get: "/applications/{application_ids.application_id}/devices/count"};
+  }
+
   // Update the end device, changing the fields specified by the field mask to the provided values.
   rpc Update(UpdateEndDeviceRequest) returns (EndDevice) {
     option (google.api.http) = {

pkg/identityserver/bunstore/end_device_store.go

@@ -439,7 +439,9 @@ func (s *endDeviceStore) CountEndDevices(ctx context.Context, ids *ttnpb.Applica
 		by = s.selectWithID(ctx, ids.GetApplicationId())
 	}
 
-	selectQuery := s.newSelectModel(ctx, &EndDevice{}).Apply(by)
+	selectQuery := s.newSelectModel(ctx, &EndDevice{}).
+		Apply(by).
+		Apply(selectWithFilterFromContext(ctx))
 
 	// Count the total number of results.
 	count, err := selectQuery.Count(ctx)

pkg/identityserver/end_device_registry.go

@@ -332,6 +332,34 @@ func (is *IdentityServer) listEndDevices(ctx context.Context, req *ttnpb.ListEnd
 	return devs, nil
 }
 
+func (is *IdentityServer) countEndDevices(
+	ctx context.Context, req *ttnpb.CountEndDevicesRequest,
+) (*ttnpb.CountEndDevicesResponse, error) {
+	if err := rights.RequireApplication(
+		ctx, req.GetApplicationIds(), ttnpb.Right_RIGHT_APPLICATION_DEVICES_READ,
+	); err != nil {
+		return nil, err
+	}
+	if req.Filters != nil {
+		for _, filter := range req.Filters {
+			if _, ok := filter.GetField().(*ttnpb.ListEndDevicesRequest_Filter_UpdatedSince); ok {
+				ctx = store.WithFilter(ctx, "updated_at", filter.GetUpdatedSince().AsTime().Format(time.RFC3339Nano))
+			}
+		}
+	}
+	var count uint64
+	err := is.store.Transact(ctx, func(ctx context.Context, st store.Store) (err error) {
+		count, err = st.CountEndDevices(ctx, req.GetApplicationIds())
+		return err
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &ttnpb.CountEndDevicesResponse{
+		Count: count,
+	}, nil
+}
+
 func (is *IdentityServer) setFullEndDevicePictureURL(ctx context.Context, dev *ttnpb.EndDevice) {
 	bucketURL := is.configFromContext(ctx).EndDevicePicture.BucketURL
 	if bucketURL == "" {
@@ -591,6 +619,12 @@ func (dr *endDeviceRegistry) Delete(ctx context.Context, req *ttnpb.EndDeviceIde
 	return dr.deleteEndDevice(ctx, req)
 }
 
+func (dr *endDeviceRegistry) Count(
+	ctx context.Context, req *ttnpb.CountEndDevicesRequest,
+) (*ttnpb.CountEndDevicesResponse, error) {
+	return dr.countEndDevices(ctx, req)
+}
+
 func (reg *endDeviceBatchRegistry) Delete(
 	ctx context.Context,
 	req *ttnpb.BatchDeleteEndDevicesRequest,

pkg/identityserver/end_device_registry_test.go

@@ -268,6 +268,113 @@ func TestEndDevicesPagination(t *testing.T) {
 	}, withPrivateTestDatabase(p))
 }
 
+func TestEndDevicesCount(t *testing.T) {
+	p := &storetest.Population{}
+
+	usr1 := p.NewUser()
+	app1 := p.NewApplication(usr1.GetOrganizationOrUserIdentifiers())
+	for range 5 {
+		p.NewEndDevice(app1.GetIds())
+	}
+
+	key, _ := p.NewAPIKey(usr1.GetEntityIdentifiers(), ttnpb.Right_RIGHT_ALL)
+	creds := rpcCreds(key)
+
+	readKey, _ := p.NewAPIKey(usr1.GetEntityIdentifiers(), ttnpb.Right_RIGHT_APPLICATION_DEVICES_READ)
+	readCreds := rpcCreds(readKey)
+
+	t.Parallel()
+	a, ctx := test.New(t)
+
+	testWithIdentityServer(t, func(_ *IdentityServer, cc *grpc.ClientConn) {
+		reg := ttnpb.NewEndDeviceRegistryClient(cc)
+
+		t.Run("Permission denied without credentials", func(_ *testing.T) { // nolint:paralleltest
+			_, err := reg.Count(ctx, &ttnpb.CountEndDevicesRequest{
+				ApplicationIds: app1.GetIds(),
+			})
+			if a.So(err, should.NotBeNil) {
+				a.So(errors.IsPermissionDenied(err), should.BeTrue)
+			}
+		})
+
+		t.Run("Count with read credentials", func(_ *testing.T) { // nolint:paralleltest
+			resp, err := reg.Count(ctx, &ttnpb.CountEndDevicesRequest{
+				ApplicationIds: app1.GetIds(),
+			}, readCreds)
+			if a.So(err, should.BeNil) && a.So(resp, should.NotBeNil) {
+				a.So(resp.Count, should.Equal, uint64(5))
+			}
+		})
+
+		t.Run("Count after adding a device", func(_ *testing.T) { // nolint:paralleltest
+			_, err := reg.Create(ctx, &ttnpb.CreateEndDeviceRequest{
+				EndDevice: &ttnpb.EndDevice{
+					Ids: &ttnpb.EndDeviceIdentifiers{
+						ApplicationIds: app1.GetIds(),
+						DeviceId:       "count-test-dev",
+					},
+				},
+			}, creds)
+			a.So(err, should.BeNil)
+
+			resp, err := reg.Count(ctx, &ttnpb.CountEndDevicesRequest{
+				ApplicationIds: app1.GetIds(),
+			}, readCreds)
+			if a.So(err, should.BeNil) && a.So(resp, should.NotBeNil) {
+				a.So(resp.Count, should.Equal, uint64(6))
+			}
+		})
+
+		t.Run("Count after deleting a device", func(_ *testing.T) { // nolint:paralleltest
+			_, err := reg.Delete(ctx, &ttnpb.EndDeviceIdentifiers{
+				ApplicationIds: app1.GetIds(),
+				DeviceId:       "count-test-dev",
+			}, creds)
+			a.So(err, should.BeNil)
+
+			resp, err := reg.Count(ctx, &ttnpb.CountEndDevicesRequest{
+				ApplicationIds: app1.GetIds(),
+			}, readCreds)
+			if a.So(err, should.BeNil) && a.So(resp, should.NotBeNil) {
+				a.So(resp.Count, should.Equal, uint64(5))
+			}
+		})
+
+		t.Run("Count with updated_since filter", func(_ *testing.T) { // nolint:paralleltest
+			// Filter by 1 hour ago - all devices should match.
+			resp, err := reg.Count(ctx, &ttnpb.CountEndDevicesRequest{
+				ApplicationIds: app1.GetIds(),
+				Filters: []*ttnpb.ListEndDevicesRequest_Filter{
+					{
+						Field: &ttnpb.ListEndDevicesRequest_Filter_UpdatedSince{
+							UpdatedSince: timestamppb.New(time.Now().Add(-time.Hour)),
+						},
+					},
+				},
+			}, readCreds)
+			if a.So(err, should.BeNil) && a.So(resp, should.NotBeNil) {
+				a.So(resp.Count, should.Equal, uint64(5))
+			}
+
+			// Filter by now - no devices should match.
+			resp, err = reg.Count(ctx, &ttnpb.CountEndDevicesRequest{
+				ApplicationIds: app1.GetIds(),
+				Filters: []*ttnpb.ListEndDevicesRequest_Filter{
+					{
+						Field: &ttnpb.ListEndDevicesRequest_Filter_UpdatedSince{
+							UpdatedSince: timestamppb.New(time.Now()),
+						},
+					},
+				},
+			}, readCreds)
+			if a.So(err, should.BeNil) && a.So(resp, should.NotBeNil) {
+				a.So(resp.Count, should.Equal, uint64(0))
+			}
+		})
+	}, withPrivateTestDatabase(p))
+}
+
 func TestEndDevicesBatchOperationsPermissions(t *testing.T) {
 	t.Parallel()
 	a, ctx := test.New(t)