feat: update metadata rework

rfcs/inactive_users/user_and_organization_meta_update.md

@@ -0,0 +1,24 @@
+# User/Organization metadata update
+## Overview and Motivation
+When user or organization metadata is updated, the Service should track audiences with assigned metadata.
+For each assigned meta hash always exists a single `audience`, but there is no list of `audiences` assigned to the user or organization.
+
+To achieve this ability, I advise these updates: 
+
+## Audience lists
+Audiences stored in sets with names created from `USERS_AUDIENCE` or `ORGANISATION_AUDIENCE` constants and `Id`
+(e.g.: `{ms-users}10110110111!audiences`). Both keys contain `audience` names that are currently have assigned values.
+
+The `audience` list will be updated on each update of the metadata.
+
+## Metadata Handling classes
+Service logic is updated to use 2 specific classes that will perform all CRUD operations on User or Organization metadata.
+
+* Classes located in: `utils/metadata/{user|organization}.js`.
+* Both classes use same [Redis backend](#redis-metadata-backend-class).
+
+## Redis Metadata Backend class
+The class performs all work on metadata using Redis DB as a backend.
+
+## Notice
+* All User or Organization metadata operations should be performed using Provided classes otherwise, audiences won't be tracked.

src/actions/activate.js

@@ -6,7 +6,8 @@ const jwt = require('../utils/jwt.js');
 const { getInternalData } = require('../utils/userData');
 const getMetadata = require('../utils/get-metadata');
 const handlePipeline = require('../utils/pipeline-error');
-const setMetadata = require('../utils/update-metadata');
+const UserMetadata = require('../utils/metadata/user');
+
 const {
   USERS_INDEX,
   USERS_DATA,
@@ -19,7 +20,7 @@ const {
   USERS_USERNAME_FIELD,
   USERS_ACTION_ACTIVATE,
   USERS_ACTIVATED_FIELD,
-} = require('../constants.js');
+} = require('../constants');
 
 // cache error
 const Forbidden = new HttpStatusError(403, 'invalid token');
@@ -121,19 +122,6 @@ async function activateAccount(data, metadata) {
   const userKey = redisKey(userId, USERS_DATA);
   const { defaultAudience, service } = this;
   const { redis } = service;
-
-  // if this goes through, but other async calls fail its ok to repeat that
-  // adds activation field
-  await setMetadata.call(service, {
-    userId,
-    audience: defaultAudience,
-    metadata: {
-      $set: {
-        [USERS_ACTIVATED_FIELD]: Date.now(),
-      },
-    },
-  });
-
   // WARNING: `persist` is very important, otherwise we will lose user's information in 30 days
   // set to active & persist
   const pipeline = redis
@@ -143,6 +131,10 @@ async function activateAccount(data, metadata) {
     .persist(userKey)
     .sadd(USERS_INDEX, userId);
 
+  UserMetadata
+    .using(userId, defaultAudience, pipeline)
+    .update(USERS_ACTIVATED_FIELD, Date.now());
+
   if (alias) {
     pipeline.sadd(USERS_PUBLIC_INDEX, userId);
   }

src/actions/alias.js

@@ -7,9 +7,10 @@ const isBanned = require('../utils/is-banned');
 const DetailedHttpStatusError = require('../utils/detailed-error');
 const key = require('../utils/key');
 const handlePipeline = require('../utils/pipeline-error');
+const UserMetadata = require('../utils/metadata/user');
+
 const {
   USERS_DATA,
-  USERS_METADATA,
   USERS_ALIAS_TO_ID,
   USERS_ID_FIELD,
   USERS_ALIAS_FIELD,
@@ -71,10 +72,12 @@ async function assignAlias({ params }) {
       return Promise.reject(err);
     }
 
-    const pipeline = redis.pipeline([
-      ['hset', key(userId, USERS_DATA), USERS_ALIAS_FIELD, alias],
-      ['hset', key(userId, USERS_METADATA, defaultAudience), USERS_ALIAS_FIELD, JSON.stringify(alias)],
-    ]);
+    const pipeline = redis.pipeline();
+
+    pipeline.hset(key(userId, USERS_DATA), USERS_ALIAS_FIELD, alias);
+    UserMetadata
+      .using(userId, defaultAudience, pipeline)
+      .update(USERS_ALIAS_FIELD, JSON.stringify(alias));
 
     if (activeUser) {
       pipeline.sadd(USERS_PUBLIC_INDEX, username);

src/actions/ban.js

@@ -4,9 +4,10 @@ const mapValues = require('lodash/mapValues');
 const redisKey = require('../utils/key.js');
 const { getInternalData } = require('../utils/userData');
 const handlePipeline = require('../utils/pipeline-error');
+const UserMetadata = require('../utils/metadata/user');
+
 const {
-  USERS_DATA, USERS_METADATA,
-  USERS_BANNED_FLAG, USERS_TOKENS, USERS_BANNED_DATA,
+  USERS_DATA, USERS_BANNED_FLAG, USERS_TOKENS, USERS_BANNED_DATA,
 } = require('../constants.js');
 
 // helper
@@ -25,26 +26,32 @@ function lockUser({
       remoteip: remoteip || '',
     },
   };
+  const pipeline = redis.pipeline();
+
+  pipeline.hset(redisKey(id, USERS_DATA), USERS_BANNED_FLAG, 'true');
+  // set .banned on metadata for filtering & sorting users by that field
+  UserMetadata
+    .using(id, defaultAudience, pipeline)
+    .updateMulti(mapValues(data, stringify));
+  pipeline.del(redisKey(id, USERS_TOKENS));
 
-  return redis
-    .pipeline()
-    .hset(redisKey(id, USERS_DATA), USERS_BANNED_FLAG, 'true')
-    // set .banned on metadata for filtering & sorting users by that field
-    .hmset(redisKey(id, USERS_METADATA, defaultAudience), mapValues(data, stringify))
-    .del(redisKey(id, USERS_TOKENS))
-    .exec();
+  return pipeline.exec();
 }
 
 function unlockUser({ id }) {
   const { redis, config } = this;
   const { jwt: { defaultAudience } } = config;
+  const pipeline = redis.pipeline();
 
-  return redis
-    .pipeline()
-    .hdel(redisKey(id, USERS_DATA), USERS_BANNED_FLAG)
-    // remove .banned on metadata for filtering & sorting users by that field
-    .hdel(redisKey(id, USERS_METADATA, defaultAudience), 'banned', USERS_BANNED_DATA)
-    .exec();
+  pipeline.hdel(redisKey(id, USERS_DATA), USERS_BANNED_FLAG);
+  // remove .banned on metadata for filtering & sorting users by that field
+  UserMetadata
+    .using(id, defaultAudience, pipeline)
+    .delete([
+      'banned',
+      USERS_BANNED_DATA,
+    ]);
+  return pipeline.exec();
 }
 
 /**

src/actions/organization/delete.js

@@ -3,6 +3,7 @@ const snakeCase = require('lodash/snakeCase');
 const redisKey = require('../../utils/key');
 const handlePipeline = require('../../utils/pipeline-error');
 const { checkOrganizationExists, getInternalData } = require('../../utils/organization');
+const OrganizationMetadata = require('../../utils/metadata/organization');
 const {
   ORGANIZATIONS_DATA,
   ORGANIZATIONS_METADATA,
@@ -32,11 +33,15 @@ async function deleteOrganization({ params }) {
   const organizationMembersListKey = redisKey(organizationId, ORGANIZATIONS_MEMBERS);
   const organizationMembersIds = await redis.zrange(organizationMembersListKey, 0, -1);
   const organization = await getInternalData.call(this, organizationId);
+  const organizationMetadata = new OrganizationMetadata(redis);
 
   const pipeline = redis.pipeline();
 
   pipeline.del(redisKey(organizationId, ORGANIZATIONS_DATA));
   pipeline.del(redisKey(organizationId, ORGANIZATIONS_METADATA, audience));
+  // delete organization audiences index
+  pipeline.del(organizationMetadata.audience.getAudienceKey(organizationId));
+
   pipeline.srem(ORGANIZATIONS_INDEX, organizationId);
   if (organizationMembersIds) {
     organizationMembersIds.forEach((memberId) => {

src/actions/organization/members/permission.js

@@ -5,6 +5,7 @@ const { checkOrganizationExists } = require('../../../utils/organization');
 const redisKey = require('../../../utils/key');
 const handlePipeline = require('../../../utils/pipeline-error');
 const getUserId = require('../../../utils/userData/get-user-id');
+const UserMetadata = require('../../../utils/metadata/user');
 const { ErrorUserNotMember, USERS_METADATA, ORGANIZATIONS_MEMBERS } = require('../../../constants');
 
 /**
@@ -41,7 +42,10 @@ async function setOrganizationMemberPermission({ params }) {
   permissions = JSON.stringify(permissions);
 
   const pipeline = redis.pipeline();
-  pipeline.hset(memberMetadataKey, organizationId, permissions);
+
+  UserMetadata
+    .using(userId, audience, pipeline)
+    .update(organizationId, permissions);
   pipeline.hset(redisKey(organizationId, ORGANIZATIONS_MEMBERS, userId), 'permissions', permissions);
 
   return pipeline.exec().then(handlePipeline);

src/actions/organization/members/remove.js

@@ -3,6 +3,7 @@ const redisKey = require('../../../utils/key');
 const getUserId = require('../../../utils/userData/get-user-id');
 const handlePipeline = require('../../../utils/pipeline-error');
 const { checkOrganizationExists } = require('../../../utils/organization');
+const UserMetadata = require('../../../utils/metadata/user');
 const {
   ORGANIZATIONS_MEMBERS,
   USERS_METADATA,
@@ -36,7 +37,9 @@ async function removeMember({ params }) {
   const pipeline = redis.pipeline();
   pipeline.del(memberKey);
   pipeline.zrem(redisKey(organizationId, ORGANIZATIONS_MEMBERS), memberKey);
-  pipeline.hdel(memberMetadataKey, organizationId);
+  UserMetadata
+    .using(userId, audience, pipeline)
+    .delete(organizationId);
 
   return pipeline.exec().then(handlePipeline);
 }

src/actions/register.js

@@ -9,7 +9,7 @@ const reduce = require('lodash/reduce');
 const last = require('lodash/last');
 
 // internal deps
-const setMetadata = require('../utils/update-metadata');
+const UserMetadata = require('../utils/metadata/user');
 const redisKey = require('../utils/key');
 const jwt = require('../utils/jwt');
 const isDisposable = require('../utils/is-disposable');
@@ -231,13 +231,13 @@ async function performRegistration({ service, params }) {
     commonMeta[USERS_ACTIVATED_FIELD] = Date.now();
   }
 
-  await setMetadata.call(service, {
-    userId,
-    audience,
-    metadata: audience.map((metaAudience) => ({
-      $set: Object.assign(metadata[metaAudience] || {}, metaAudience === defaultAudience && commonMeta),
-    })),
-  });
+  await UserMetadata
+    .using(userId, audience, service.redis)
+    .batchUpdate({
+      metadata: audience.map((metaAudience) => ({
+        $set: Object.assign(metadata[metaAudience] || {}, metaAudience === defaultAudience && commonMeta),
+      })),
+    });
 
   // assign alias
   if (alias) {

src/actions/remove.js

@@ -7,6 +7,7 @@ const key = require('../utils/key');
 const { getInternalData } = require('../utils/userData');
 const getMetadata = require('../utils/get-metadata');
 const handlePipeline = require('../utils/pipeline-error');
+const UserMetadata = require('../utils/metadata/user');
 const {
   USERS_INDEX,
   USERS_PUBLIC_INDEX,
@@ -92,6 +93,8 @@ async function removeUser({ params }) {
   const alias = internal[USERS_ALIAS_FIELD];
   const userId = internal[USERS_ID_FIELD];
   const resolvedUsername = internal[USERS_USERNAME_FIELD];
+  const metaAudiences = await UserMetadata.using(userId, null, redis).getAudience();
+  const userMetadata = UserMetadata.using(userId, null, transaction);
 
   if (alias) {
     transaction.hdel(USERS_ALIAS_TO_ID, alias.toLowerCase(), alias);
@@ -114,7 +117,9 @@ async function removeUser({ params }) {
 
   // remove metadata & internal data
   transaction.del(key(userId, USERS_DATA));
-  transaction.del(key(userId, USERS_METADATA, audience));
+  for (const metaAudience of metaAudiences) {
+    userMetadata.deleteMetadata(metaAudience);
+  }
 
   // remove auth tokens
   transaction.del(key(userId, USERS_TOKENS));

src/actions/updateMetadata.js

@@ -1,6 +1,5 @@
-const omit = require('lodash/omit');
 const Promise = require('bluebird');
-const updateMetadata = require('../utils/update-metadata');
+const UserMetadata = require('../utils/metadata/user');
 const { getUserId } = require('../utils/userData');
 
 /**
@@ -19,12 +18,15 @@ const { getUserId } = require('../utils/userData');
  * @apiParam (Payload) {Object} [script] - if present will be called with passed metadata keys & username, provides direct scripting access.
  *   Be careful with granting access to this function.
  */
-module.exports = function updateMetadataAction(request) {
-  return Promise
+module.exports = async function updateMetadataAction(request) {
+  const { username: _, audience, ...updateParams } = request.params;
+  const userId = await Promise
     .bind(this, request.params.username)
-    .then(getUserId)
-    .then((userId) => ({ ...omit(request.params, 'username'), userId }))
-    .then(updateMetadata);
+    .then(getUserId);
+
+  return UserMetadata
+    .using(userId, audience, this.redis)
+    .batchUpdate(updateParams);
 };
 
 module.exports.transports = [require('@microfleet/core').ActionTransport.amqp];

src/auth/oauth/utils/attach.js

@@ -1,13 +1,13 @@
 const get = require('lodash/get');
 const redisKey = require('../../../utils/key');
-const updateMetadata = require('../../../utils/update-metadata');
+const UserMetadata = require('../../../utils/metadata/user');
 const handlePipeline = require('../../../utils/pipeline-error');
 const {
   USERS_SSO_TO_ID,
   USERS_DATA,
 } = require('../../../constants');
 
-module.exports = function attach(account, user) {
+module.exports = async function attach(account, user) {
   const { redis, config } = this;
   const { id: userId } = user;
   const {
@@ -23,17 +23,18 @@ module.exports = function attach(account, user) {
   // link uid to user id
   pipeline.hset(USERS_SSO_TO_ID, uid, userId);
 
-  return pipeline.exec().then(handlePipeline)
-    .bind(this)
-    .return({
-      userId,
-      audience,
-      metadata: {
-        $set: {
-          [provider]: profile,
-        },
+  handlePipeline(await pipeline.exec());
+
+  const updateParams = {
+    metadata: {
+      $set: {
+        [provider]: profile,
       },
-    })
-    .then(updateMetadata)
-    .return(profile);
+    },
+  };
+  await UserMetadata
+    .using(userId, audience, redis)
+    .batchUpdate(updateParams);
+
+  return profile;
 };

src/auth/oauth/utils/detach.js

@@ -2,7 +2,7 @@ const Errors = require('common-errors');
 
 const get = require('../../../utils/get-value');
 const redisKey = require('../../../utils/key');
-const updateMetadata = require('../../../utils/update-metadata');
+const UserMetadata = require('../../../utils/metadata/user');
 const handlePipeline = require('../../../utils/pipeline-error');
 
 const {
@@ -30,13 +30,15 @@ module.exports = async function detach(provider, userData) {
 
   handlePipeline(await pipeline.exec());
 
-  return updateMetadata.call(this, {
-    userId,
-    audience,
+  const updateParams = {
     metadata: {
       $remove: [
         provider,
       ],
     },
-  });
+  };
+
+  return UserMetadata
+    .using(userId, audience, redis)
+    .batchUpdate(updateParams);
 };