Tutorial : create a custom CSV collector

The core folder provides an abstract class CSVCollector which can serve as the basis for quickly creating collectors that retrieve their data from CSV files.

To create such a collector you need to:

1. Create a class derived from CSVCollector
2. Create the json definition file for the data synchro source
3. Add a configuration parameter (in params.local.xml) to define the CSV file to parse
4. Register your collector in collectors/main.php

Let's create a very simple CSV collector which copies the “Person” objects (class Person)

Since the collector inherits all its behavior from the base class, the PHP code for the collector is simply:

collectors/iTopPersonCsvCollector.class.inc.php

<?php
class iTopPersonCsvCollector extends CSVCollector
{
}

iTopPersonCsvCollector.json

{
        "name": "$prefix$Synchro CSV Person",
        "description": "Synchronization of persons from CSV",
        "status": "$synchro_status$",
        "user_id": "$synchro_user$",
        "notify_contact_id": "$contact_to_notify$",
        "scope_class": "Person",
        "database_table_name": "$persons_data_table$",
        "scope_restriction": "",
        "full_load_periodicity": "$full_load_interval$",
        "reconciliation_policy": "use_primary_key",
        "action_on_zero": "create",
        "action_on_one": "update",
        "action_on_multiple": "error",
        "delete_policy": "ignore",
        "delete_policy_update": "",
        "delete_policy_retention": "0",
        "attribute_list": [
                {
                        "attcode": "cis_list",
                        "update": "0",
                        "reconcile": "0",
                        "update_policy": "master_locked",
                        "row_separator": "|",
                        "attribute_separator": ";",
                        "value_separator": ":",
                        "attribute_qualifier": "'",
                        "finalclass": "SynchroAttLinkSet",
                        "friendlyname": "cis_list"
                },
                {
                        "attcode": "email",
                        "update": "1",
                        "reconcile": "1",
                        "update_policy": "master_locked",
                        "finalclass": "SynchroAttribute",
                        "friendlyname": "email"
                },
                {
                        "attcode": "employee_number",
                        "update": "1",
                        "reconcile": "0",
                        "update_policy": "master_locked",
                        "finalclass": "SynchroAttribute",
                        "friendlyname": "employee_number"
                },
                {
                        "attcode": "first_name",
                        "update": "1",
                        "reconcile": "1",
                        "update_policy": "master_locked",
                        "finalclass": "SynchroAttribute",
                        "friendlyname": "first_name"
                },
                {
                        "attcode": "function",
                        "update": "1",
                        "reconcile": "0",
                        "update_policy": "master_locked",
                        "finalclass": "SynchroAttribute",
                        "friendlyname": "function"
                },
                {
                        "attcode": "location_id",
                        "update": "0",
                        "reconcile": "0",
                        "update_policy": "master_locked",
                        "reconciliation_attcode": "name",
                        "finalclass": "SynchroAttExtKey",
                        "friendlyname": "location_id"
                },
                {
                        "attcode": "manager_id",
                        "update": "0",
                        "reconcile": "0",
                        "update_policy": "master_locked",
                        "reconciliation_attcode": "email",
                        "finalclass": "SynchroAttExtKey",
                        "friendlyname": "manager_id"
                },
                {
                        "attcode": "mobile_phone",
                        "update": "1",
                        "reconcile": "0",
                        "update_policy": "master_locked",
                        "finalclass": "SynchroAttribute",
                        "friendlyname": "mobile_phone"
                },
                {
                        "attcode": "name",
                        "update": "1",
                        "reconcile": "1",
                        "update_policy": "master_locked",
                        "finalclass": "SynchroAttribute",
                        "friendlyname": "name"
                },
                {
                        "attcode": "notify",
                        "update": "0",
                        "reconcile": "0",
                        "update_policy": "master_locked",
                        "finalclass": "SynchroAttribute",
                        "friendlyname": "notify"
                },
                {
                        "attcode": "org_id",
                        "update": "1",
                        "reconcile": "1",
                        "update_policy": "master_locked",
                        "reconciliation_attcode": "name",
                        "finalclass": "SynchroAttExtKey",
                        "friendlyname": "org_id"
                },
                {
                        "attcode": "phone",
                        "update": "1",
                        "reconcile": "0",
                        "update_policy": "master_locked",
                        "finalclass": "SynchroAttribute",
                        "friendlyname": "phone"
                },
                {
                        "attcode": "status",
                        "update": "1",
                        "reconcile": "0",
                        "update_policy": "master_locked",
                        "finalclass": "SynchroAttribute",
                        "friendlyname": "status"
                },
                {
                        "attcode": "team_list",
                        "update": "0",
                        "reconcile": "0",
                        "update_policy": "master_locked",
                        "row_separator": "|",
                        "attribute_separator": ";",
                        "value_separator": ":",
                        "attribute_qualifier": "'",
                        "finalclass": "SynchroAttLinkSet",
                        "friendlyname": "team_list"
                }
        ],
        "user_delete_policy": "nobody",
        "url_icon": "",
        "url_application": "",
        "notify_contact_id_obsolescence_flag": ""
}

For each CSV collector you need to start its configuration by XML target with the name of your class:

conf/params.local.xml

<collector_class></collector_class>

Here are below some sample configurations:

Here is an example of configuration for a CSV collector with header inside the CSV file to import.

clé primaire;prénom;nom;org_id;téléphone;employé;email;fonction;statut
1;isaac;asimov;Demo;123456789;06543210;issac.asimov@function.io;writer;Active

You can see how to configure mapping/default values and ignored values. when mapping is not specified this is considered as implicit configuration: csv column is the iTop field to synchronize.

Then in params.local.xml, add the following entries:

conf/params.local.xml

  <iTopPersonCsvCollector>
    <csv_file>collectors/iTopPersonCsvCollector.csv</csv_file>
    <command>sed -i -e 's|isaac|ISAAC|g' collectors/iTopPersonCsvCollector.csv</command>
    <encoding>UTF-8</encoding>    
    <fields>
      <first_name>prénom</first_name>
      <primary_key>clé primaire</primary_key>
      <employee_number>employé</employee_number>
      <function>fonction</function>
      <status>statut</status>
      <phone>téléphone</phone>
      <name>nom</name>
    </fields>
    <defaults>
      <mobile_phone>9998877665544</mobile_phone>
    </defaults>
    <ignored_columns type="array">
      <ignored_attribute>fonction</ignored_attribute>
      <ignored_attribute>org_id</ignored_attribute>
    </ignored_columns>
    <nullified_attributes type="array">
      <attribute>phone</attribute>
      <attribute>employee_number</attribute>
    </nullified_attributes>
  </iTopPersonCsvCollector>

Here is an example of configuration for a CSV collector without header inside the CSV file to import. for csv fields we use index to help the collector parse the file.

clé primaire;prénom;nom;org_id;téléphone;employé;email;fonction;statut

You can see how to configure mapping/default values and ignored values.

conf/params.local.xml

  <iTopPersonCsvCollector>
    <csv_file>collectors/iTopPersonCsvCollector.csv</csv_file>
    <has_header>no</has_header>
    <fields>
      <first_name>2</first_name>
      <primary_key>1</primary_key>
      <employee_number>7</employee_number>
      <email>8</email>
      <ignored_function>9</ignored_function>
      <mobile_phone>6</mobile_phone>
      <phone>5</phone>
      <ignored_org_id>4</ignored_org_id>
      <name>3</name>
    </fields>
    <defaults>
      <org_id>Demo</org_id>
      <status>Active</status>
    </defaults>
    <ignored_columns type="array">
      <ignored_attribute>9</ignored_attribute>
      <ignored_attribute>4</ignored_attribute>
    </ignored_columns>
    <nullified_attributes type="array">
      <attribute>phone</attribute>
      <attribute>employee_number</attribute>
    </nullified_attributes>
  </iTopPersonCsvCollector>

Finally, in collectors/main.php add the following lines:

collectors/main.php

<?php
require_once(APPROOT.'collectors/iTopPersonCsvCollector.class.inc.php');
Orchestrator::AddCollector($index++, 'iTopPersonCsvCollector');

To launch the data collection and synchronization with iTop, run the following command (from the root directory where the application is installed):

php exec.php