Project Gutenberg ebook feed analysis

This sample code uses Project Gutenberg‘s feeds and performs some basic querying and analysis for potential mis-categorized books.

/*

Contributed by Adam Shirey (adam.shirey@lexisnexis.com).

Data originating from Project Gutenberg Feed's page (gutenberg.org/wiki/Gutenberg:Feeds).



To Spray as XML, download the current Project Gutenberg catalog (direct download link: http://www.gutenberg.org/feeds/catalog.rdf.zip).

Unzip catalog.rdf using Windows Explorer or a command-line zip utility.

After loading catalog.rdf to your dropzone, select the appropriate machine/IP address and local path.

Set the Row Tag to "pgterms:etext" and the label to "ecldata::in::gutenberg_catalog" and click Submit.

*/





layout_subjects := record

@nbsp;@nbsp;	string subject {xpath('dcterms:LCSH/rdf:value')};

end;

layout := record, maxlength(10240)

@nbsp;	string publisher       {xpath('dc:publisher')};

@nbsp;	string title           {xpath('dc:title')};

@nbsp;	string description     {xpath('dc:description')};

@nbsp;	string tableOfContents {xpath('dc:tableOfContents')};

@nbsp;	string creator         {xpath('dc:creator')};

@nbsp;	string friendlyTitle   {xpath('pgterms:friendlytitle')};

@nbsp;	string language        {xpath('dc:language/dcterms:ISO639-2/value')};

@nbsp;	string created         {xpath('dc:created/dcterms:W3CDTF/rdf:value')};

@nbsp;	string modified        {xpath('dc:modified/dcterms:W3CDTF/rdf:value')};

@nbsp;	string rights          {xpath('dc:rights')};

@nbsp;	string subject_lcsh    {xpath('dc:subject/dcterms:LCSH/rdf:value')}; // library of congress subject heading; see en.wikipedia.org/wiki/Library_of_Congress_Subject_Headings

@nbsp;	string subject_lcc     {xpath('dc:subject/dcterms:LCC/rdf:value')}; // library of congress classification; see en.wikipedia.org/wiki/Library_of_Congress_Classification

@nbsp;	dataset(layout_subjects) subjects {xpath('dc:subject/rdf:Bag/rdf:li')};

end;



gutenberg := dataset( '~ecldata::in::gutenberg_catalog', layout, xml('rdf:RDF/pgterms:etext') );



// determine the top ten 'creator' values

creator_info := table( gutenberg, {creator, n := count(group)}, creator );

creator_top10 := choosen( sort(creator_info,-n), 10 );

output(creator_top10, named('top10_creators'));



// a sampling of titles based on their class and subclass

output( gutenberg(subject_lcc='BQ'), named('buddhism') ); // Philosophy, Psychology, Relegion - Buddhism

output( gutenberg(subject_lcc='DJ'), named('netherlands_history') ); // World History - Netherlands (Holland)

output( gutenberg(subject_lcc='GV'), named('recreation') ); // Geography, Anthropology, Recreation - Recreation, Leisure

output( gutenberg(subject_lcc='RB'), named('pathology') ); // Medicine - Pathology

output( gutenberg(subject_lcc='TR'), named('photography') ); // Technology - Photography

output( gutenberg(subject_lcc[1]='E' and subject_lcc[2..] between '456' and '655'), named('us_civil_war') ); // American History -- Civil War period, 1861-1865



// using HPCC to detect potential mis-categorized books, find records with a Library of Congress Classification of BF (psychology) that don't include the 'psych' prefix

miscategorized := gutenberg(

@nbsp;	subject_lcc='BF', // records in the Philosophy, Psychology, Religion class, Psychology subclass

@nbsp;	not regexfind( '\\bpsych\\B', title, NOCASE ), // but the title doesn't contain a 'psych-' word (case-insensitive)

@nbsp;	not regexfind( '\\bpsych\\B', description, NOCASE ), // nor does the description

@nbsp;	not exists( subjects( regexfind( '\\bpsych\\B', subject, NOCASE ) ) ) // nor do any of the subjects

);

output( miscategorized, named('miscategorized_psychology') );