Browse plugins

plugin-dedupe

Dedupe records in a sheet via a sheet level custom action.

Install
npm i @flatfile/plugin-dedupe
Package:
@flatfile/plugin-dedupe 29 installs
Dependencies
@flatfile/plugin-job-handler@^0.6.1, @flatfile/util-common@^1.4.1

This plugin dedupes records in a Sheet via a Sheet-level action.

Event Type: listener.on('job:ready')

When embedding Flatfile, this plugin should be deployed in a server-side listener. Learn more

Parameters

jobOperation - string - (required)

The jobOperation parameter specifies the name of the job operation to match on.

opt.on - string

The on parameter specifies which field key to match on.

opt.keep - 'first' | 'last'

The keep option lets you choose whether to keep the first or last duplicate record.

opt.custom - function()

The custom parameter accepts a custom dedupe function. This will override the keep parameter.

opt.debug - boolean

The debug parameter lets you toggle on/off helpful debugging messages for development purposes.

API Calls

  • api.records.get
  • api.jobs.ack
  • api.records.delete
  • api.jobs.fail
  • api.jobs.complete

Usage

An action with the operation name of "dedupe-email" must be configured on a Sheet for the plugin to be triggered.

npm i @flatfile/plugin-dedupe
import { dedupePlugin } from "@flatfile/plugin-dedupe";
import { Flatfile } from '@flatfile/api'

export const contactsSheet: Flatfile.SheetConfig = {
  name: 'Contacts',
  slug: 'contacts',
  fields: [
    {
      key: 'firstName',
      type: 'string',
      label: 'First Name',
    },
    {
      key: 'lastName',
      type: 'string',
      label: 'Last Name',
    },
    {
      key: 'email',
      type: 'string',
      label: 'Email',
    }
  ],
  // Add a Sheet-level action here for the dedupe plugin
  actions: [
    {
      operation: "dedupe-email",
      mode: "background",
      label: "Dedupe emails",
      description: "Remove duplicate emails"
    }
  ]
}

JavaScript

listener.js

// common usage
// Keep the last record encountered (from top to bottom) based on the`email` field key.
// Must have a Sheet-level action specified with the operation name `dedupe-email`
listener.use(
  dedupePlugin("dedupe-email", {
    on: "email",
    keep: "last",
  })
);

// user specified dedupe function must return a list a record id's for deletion
listener.use(
  dedupePlugin("dedupe-email", {
    custom: (records) => {
      let uniques = new Set();
      let toDelete = [];

      records.forEach(record => {
        const { value } = record.values["email"];
        if (uniques.has(value)) {
          toDelete.push(record.id);
        } else {
          uniques.add(value);
        }
      });

      return toDelete;
    },
  })
);

TypeScript

listener.ts

// common usage
// Keep the last record encountered (from top to bottom) based on the`email` field key.
// Must have a Sheet-level action specified with the operation name `dedupe-email`
listener.use(
  dedupePlugin("dedupe-email", {
    on: "email",
    keep: "last",
  })
);

// user specified dedupe function must return a list a record id's for deletion
listener.use(
  dedupePlugin("dedupe-email", {
    custom: (records: Flatfile.RecordsWithLinks) => {
      let uniques = new Set();
      let toDelete = [];

      records.forEach(record => {
        const { value } = record.values["email"];
        if (uniques.has(value)) {
          toDelete.push(record.id);
        } else {
          uniques.add(value);
        }
      });

      return toDelete;
    },
  })
);