How to use an installed public package in Dataform?

[tomlag]

I installed this package in dataform successfully
https://www.npmjs.com/package/libphonenumber-js

This is how it looks the package.json

{
  "name": "vx_operation",
  "dependencies": {
    "@dataform/core": "3.0.2",
    "libphonenumber-js":"1.11.7"
  }
}

This is how it looks the require statement. And I'm getting the error "Unexpected token 'export'"

Do you know what could be?

I searched the dataform documentation several times, including the googlecloudcommunity and youtube. I didn't find examples of installing external libraries. 

Thanks

[tomlag · Accepted solution]

This is a "module" type package (ES) which requires to async. 
I changed the package to google-libphonenumber and now is working fine to require the package in the standard way that dataform needs.

[OneMoreSteph]

Hi Tom,

I'm encountering the same issue while trying to import a public NPM package in Dataform. Would you mind sharing how your solution is implemented in your `package.json` (dependencies block) and `.js` (the file where the package is used) file ? Thanks

[tomlag]

Hi @OneMoreSteph, although I made it work, I really needed to use library to validate phones inside a table in dataform. So the dataform installation didn't work for me. Instead I uploaded the library in google storage and created a routine using dataform

config {
  type: "operations",
  schema: "vx_data_distributor"
}

CREATE OR REPLACE FUNCTION myMainProject.phoneNumberToE164(phone STRING, countryCode STRING) RETURNS STRING
LANGUAGE js
OPTIONS (library='gs://myMainBucket/libphonenumber.js') -- https://www.npmjs.com/package/google-libphonenumber v3.2.38

AS r"""
    const PNF = libphonenumber.PhoneNumberFormat;
    const phoneUtil = libphonenumber.PhoneNumberUtil.getInstance();

    // Check for null or empty input and return null
    if (!phone) return null;

    // Set default country code if not provided
    countryCode = countryCode || 'AR';

    // Trim leading and trailing spaces, then remove all non-numeric characters except "+" if it's the first character after trimming
    let cleanedPhone = phone.trim().replace(phone.trim()[0] === "+" ? /[^\d+]/g : /[^\d]/g, '');

    // Seven digit phone numbers are deprecated
    if (cleanedPhone.length <= 7 || cleanedPhone.length >= 17) return null;

    let number;
    let national;
    try {
        number = phoneUtil.parse(cleanedPhone, countryCode);
        national = number.getNationalNumber().toString();
    } catch (e) {
        // Log the error or handle it as needed
        console.error('Error parsing phone number:', e);
        return null;
    }
    
    // Handle Argentinian phone numbers (country code 54)
    if (number.getCountryCode() === 54) {
        // Adjust national number if needed
        if (national.startsWith('15')) {
            national = national.substring(2);
        } else if (national.startsWith('1115')) {
            national = '11' + national.substring(4);
        } else if (national.startsWith('01115')) {
            national = '11' + national.substring(5);
        }

        // Default Buenos Aires if area is missing
        if (national.length < 10) {
            cleanedPhone = '54911' + national;
        }
        else if (national[0] != '9') {
            cleanedPhone = '549' + national;
        }
    }

    // Attempt to format the number in E164 format
    try {
        return phoneUtil.format(phoneUtil.parse(cleanedPhone, countryCode), PNF.E164);;
    } catch (e) {
        // Log the error or handle it as needed
        console.error('Error formatting phone number:', e);
        return null;
    }
""";

-- tests
WITH test_cases AS (
    -- General cases
    SELECT "" AS input, "_empty" AS test_case, null AS expected_result
    UNION ALL SELECT null, "_null", null
    UNION ALL SELECT "    ", "_espacios", null
    UNION ALL SELECT "5265895", "_7_digitos", null
    UNION ALL SELECT "52658952", "_8_digitos", "+5491152658952"
    UNION ALL SELECT "4140-2086", "_8_digitos_casa", "+5491141402086"
    UNION ALL SELECT "526589588", "_9_digitos", "+54911526589588"
    UNION ALL SELECT "(351)206-2615", "_10_digitos", "+5493512062615"
    UNION ALL SELECT "1145671234", "_10_digitos_inicia_11", "+5491145671234"
    UNION ALL SELECT "1545671234", "_10_digitos_inicia_15", "+5491145671234"
    UNION ALL SELECT "2325671234", "_10_digitos_no_inicia_15", "+5492325671234"
    UNION ALL SELECT "01123456789", "_11_digitos_inicia_0", "+5491123456789"
    UNION ALL SELECT "12345678901", "_11_digitos_no_inicia_0", "+54912345678901"
    UNION ALL SELECT "154567123456", "_12_digitos_inicia_15", "+5494567123456"
    UNION ALL SELECT "541123456789", "_12_digitos_inicia_54_no_549", "+5491123456789"
    UNION ALL SELECT "549123456789", "_12_digitos_inicia_549", "+549123456789"
    UNION ALL SELECT "0111567890123", "_13_digitos_inicia_01115", "+5491167890123"
    UNION ALL SELECT "0113456789012", "_13_digitos_no_inicia_01115", "+549113456789012"
    UNION ALL SELECT "+549987654321", "_14_digitos_inicia_549", "+549987654321"
    UNION ALL SELECT "541987654321", "_12_digitos_inicia_541", "+5491987654321"
    UNION ALL SELECT "12345678901234", "_14_digitos_no_inicia_549_ni_54", "+54912345678901234"
    UNION ALL SELECT "(011) 4567-8901", "_11_digitos_formato_vario", "+5491145678901"
    UNION ALL SELECT "+54 9 11 1234-5678", "_formato_internacional_ya_converted", "+5491112345678"

    -- International Prefixes
    UNION ALL SELECT "+5491126589588", "_13_digitos_549_prefix_international", "+5491126589588"
    UNION ALL SELECT "+54911265895889", "_14_digitos_549_prefix_international", "+54911265895889"

    -- Cases starting with '15'
    UNION ALL SELECT "1526589588", "_10_digitos_15_prefix", "+5491126589588"
    UNION ALL SELECT "23890433", "_8_digitos", "+5491123890433"
    UNION ALL SELECT "1523890433", "_10_digitos_15_prefix", "+5491123890433"

    -- Cases starting with '11'
    UNION ALL SELECT "1152658958", "_11_digitos_11_prefix", "+5491152658958"
    UNION ALL SELECT "01152658958", "_11_digitos_11_prefix", "+5491152658958"

    -- Cases starting with '011'
    UNION ALL SELECT "011526589588", "_12_digitos_011_prefix", "+54911526589588"

    -- Cases interior prefix
    UNION ALL SELECT "3512062615", "_9_digitos_246_prefix", "+5493512062615"
    UNION ALL SELECT "24652658952", "_9_digitos_246_prefix", "+54924652658952"
    UNION ALL SELECT "24611526589588", "_14_digitos_246_prefix", "+54924611526589588"
    UNION ALL SELECT "2465265895889", "_13_digitos_246_prefix_2", "+5492465265895889"
    UNION ALL SELECT "353715586870", "_12_digitos_3537_prefix_2", "+5493537586870"
    UNION ALL SELECT "3512062615", "_11_digitos_351_prefix_2", "+5493512062615"


    -- Cases starting with '54'
    UNION ALL SELECT "5411526589588", "_12_digitos_54_prefix", "+54911526589588"

    -- Cases starting with '549'
    UNION ALL SELECT "91123890833", "_9_prefix", "+5491123890833"
    UNION ALL SELECT "01123890833", "_011_prefix", "+5491123890833"
    UNION ALL SELECT "111523890833", "_1115_prefix", "+5491123890833"
    UNION ALL SELECT "0111523890833", "_01115_prefix", "+5491123890833"
    UNION ALL SELECT "54952658952", "_9_digitos_549_prefix", "+54954952658952"
    UNION ALL SELECT "5491526589588", "_13_digitos_549_prefix", "+5491526589588"
    UNION ALL SELECT "54911526589588", "_14_digitos_54911_prefix", "+54911526589588"
    UNION ALL SELECT "5495265895889", "_13_digitos_549_prefix_2", "+5495265895889"
    UNION ALL SELECT "549115265895889", "_15_digitos_54911_prefix", "+549115265895889"

    -- Cases with valid area codes
    UNION ALL SELECT "1126589588", "_10_digitos_valid_buenos_aires", "+5491126589588"
    UNION ALL SELECT "35126589588", "_11_digitos_valid_cordoba", "+54935126589588"
    UNION ALL SELECT "29726589588", "_11_digitos_valid_comodoro_rivadavia", "+54929726589588"

    -- Internatinal numbers
    UNION ALL SELECT "+34722534819", "_11_digitos_internacional", "+34722534819"

)
SELECT 
    test_case,
    input, 
    vx_data_distributor.phoneNumberToE164(input,"AR") AS result, 
    expected_result,
    CASE
        WHEN vx_data_distributor.phoneNumberToE164(input,"AR") = expected_result 
             OR (vx_data_distributor.phoneNumberToE164(input,"AR") IS NULL AND expected_result IS NULL) 
        THEN 'Pass'
        ELSE 'Fail'
    END AS evaluation,
FROM 
    test_cases;

[OneMoreSteph]

Hi @tomlag ,

I did not expect such an answer. Thank you for sharing your complete setup. I will dig into it. 🙂