Example 33.1. Deduplication

table 'input_data':
    id imie         nazwisko    ulica            nr_domu data_urodzenia kod_pocztowy plec
    1 "Joanna"      "Spyt"      "Sorkratesa"     "15"   "1985-09-24"    "86-126"     "F"
    2 "Michał"      "Jarosz"    "Sokołowska"     None   "1995-12-29"    "90-691"     "M"
    3 "Jakub"       "Sokół"     "Piaskowa"       "72"   None            "36-912"     "M"
    4 "Pola"        "Łada"      "Łąkocińska"     "79"   "1988-05-28"    "12-726"     "F"
    5 "Anastazja"   "Zborowska" "Gorlicka"       "76"   "1978-07-15"    "30-452"     "F"
    6 "Jakub"       "Zbrowski"  "Niezapominajki" None   "1977-10-08"    "48-371"     "M"
    7 "Julia"       "Jóźwiak"   "Styrska"        "57"   None            "58-855"     "F"
    8 "Aleksandra"  "Tytuła"    "Buńczuk"        "8"    "2005-07-17"    None         "F"
    9 "Joanna"      "Sprot"     "Małcużyńskiego" "94"   None            "27-237"     "F"
    10 "Mateusz"    "Saks"      "Prudnicka"      "80"   "1973-01-27"    "34-828"     "M"
    11 "Piotr"      "Przyborowski" "Biedronki"   "52"   None            "53-123"     "M"
    12 "Maja"       "Stasiak"   "Racjonalizacji" "37"   None            "01-344"     "F"
    # duplicates
    13 "Joana"      "Spyd"      "Sorkratesa"     "15"   "1985-09-25"    "86-126"     "F"
    14 "Micał"      "Jalosz"    "Sokolowska"     None   "1995-12-29"    "90-691"     "M"
    15 "akub"       "Sokol"     "Pieskowa"       "72"   None            "36-912"     "M"

inputTable = 'input_data'
pd = PhysicalData(inputTable)
outputTable = inputTable + "_out"


#------ algorithm settings ------
as = WeightedMatchingAlgorithmSettings()
as.cutOff = 3.0
as.setIgnoreNullsInGroup(0)
as.setCaseSensitive(0)

#------ function settings ------
fs = WeightedMatchingFunctionSettings()
fs.setAlgorithmSettings(as)
fs.setLogicalData(LogicalData(pd))
fs.getAttributeUsageSet().getAttribute('id').setUsage(MatchingUsage.id)
fs.getAttributeUsageSet().getAttribute('plec').setUsage(MatchingUsage.block)


#------ mining task ------
mt = MatchingTask()
mt.setTarget(DataObjectSet(outputTable))
mt.setPhysicalDataName(inputTable+'_pd')
mt.setMatchingFunctionSettingsName(inputTable+'_fs')


save(inputTable+'_pd', pd)
save(inputTable + '_fs', fs)
save(inputTable+'_mt', mt)

execute(inputTable + '_mt')

print 'Done'
from specialized.dataquality import *
matching_results(inputTable + '_mt', inputTable)
    

This script creates an Excel spreadsheet report.