Dashboard v0.3 - Digital History

Stage 3:

This version is now closed. Please see Dashboard v0.4 for more recent developments.

This builds upon the Dashboard v0.1 (Fall 2008) and Dashboard v0.2 (Spring 2009).

a Video of the Dashboard can be seen on the NiCHE Digital Infrastructure site: http://niche.uwo.ca/node/2005

The goals of this section are:

to integrate the two feeds and display them both on the LCD screen at the same time: one on the top row, one on the bottom.
include a timer, using the local computer's clock, that will refresh the data at set intervals.

1) The two feeds have been combined into one resource pool from which the program selects an entry and displays it to the screen. Only one entry at a time is displayed to prevent clutter.

2) This has been put off while the code is transferred from Phidgets to Arduino.

To do: Google Analytics data is not automatically called when the program starts; though the NiCHE RSS feed is. No timer yet imposed. Exact data we want to use also not decided - though this is not crucial at this point.

Future sections will:

Create a version of the Dashboard that uses Arduino rather than Phidgets.

Code Modules

Downloading Data

NiCHERSSv001.py - Parses the NiCHENews RSS feed (created by Drupal), and saves the titles to NiCHERSS.txt
NiCHE-GA-RSS.py - Parses Google Groups Feed of Google Analytics XML files and saves the XML files to NiCHE-GA-RSS.txt

Organizing Data

GARSSParsev001.py - Parses NiCHE-GA-RSS.txt for desired visitor data, compiles it into weekly totals and saves to a Python List

Displaying Data

RSStoLCDv003.py - Parses NiCHERSS.txt and calls values from List containing visitor data. Displays random titles to Phidgets LCD Screen

NiCHERSSv001.py

# NiCHERSSv001.py 
# By Adam Crymble (acrymbl@uwo.ca)
# modified from original in Programming Historian: http://niche.uwo.ca/programming-historian/index.php/Working_with_files_and_web_pages
#!/usr/bin/python

import urllib2

#gets the RSS feed from the below address
url = 'http://niche.uwo.ca/feed/nichenews'
response = urllib2.urlopen(url)
html = response.read()

#creates a file and writes titles to it
f = open('NiCHE-RSS.txt', 'w')
titleOnly = ''
x = 0
itemList = []

#creates substrings of the titles and saves to txt file
while html.find("</title>") != -1:
    titleLoc = html.find("</title>")
    titleOnly = html[html.find("<title>")+7:titleLoc]
    html = html[titleLoc  + 8:]
    print titleOnly
    itemList.append(titleOnly)
    itemCount = len(itemList)
    if x < 10:
        f.write("Feed-Item-0" + str(x) + " " + titleOnly + "\n")
    else:
        f.write("Feed-Item-" + str(x) + " " + titleOnly + "\n")
    x += 1
f.close

NiCHE-GA-RSS.py

# NiCHE-GA-RSS.py 
# By Adam Crymble (acrymbl@uwo.ca)
# modified from original in Programming Historian: http://niche.uwo.ca/programming-historian/index.php/Working_with_files_and_web_pages
#!/usr/bin/python

import urllib
import urllib2

# Pretends the request is coming from Firefox to avoid a Forbidden 403 message and downloads the RSS feed of GA data
url = 'http://groups.google.com/group/niche_dashboard/feed/rss_v2_0_msgs.xml?=15'
user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
values = {'name' : 'Adam Crymble',
          'location' : 'London',
          'language' : 'Python' }
headers = { 'User-Agent' : user_agent }

data = urllib.urlencode(values)
req = urllib2.Request(url, data, headers)
response = urllib2.urlopen(req)
the_page = response.read()

idOnly = ''
x = 0
idList = []

# Parses out only the part of the URL that is unique to each item and saves it to a list.
while the_page.find("<guid") != -1:

    URLid = the_page.find("</guid>")
    idOnly = the_page[the_page.find("<guid")+25:URLid]
    idOnly = idOnly[idOnly.find("show_docid=")+11:]
    the_page = the_page[URLid  + 25:]
    print idOnly
    idList.append(idOnly)
    x += 1
x = 0

print idList

# Creates the URL of the XML file from the list item and a stable URL. Then downloads the XML file and saves it to a TXT file.
f = open('NiCHE-GA-RSS.txt', 'w')

for items in idList:
    createURL = 'http://17170076143844878255-a-g.googlegroups.com/attach/' + idList[x] + '//petit-nord_20090315-20090414.xml?gda=PuQAmUYAAADwUmeXcxxJFEpiykf9_Yl7ADxPFwpLp7HtP1Vk3qu2LDAmTJkYuPIfvEFpTeovnQ9x40jamwa1UURqDcgHarKEE-Ea7GxYMt0t6nY0uV5FIQ&view=1&part=2'
    user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
    values = {'name' : 'Adam Crymble',
               'location' : 'London',
               'language' : 'Python' }
    headers = { 'User-Agent' : user_agent }

    data = urllib.urlencode(values)
    req = urllib2.Request(createURL, data, headers)
    response = urllib2.urlopen(req)
    the_XML = response.read()
    if x < 10:
        f.write("XML-Item-0" + str(x) + " " + idList[x] + "\n")
        f.write(the_XML + "\n")
    else:
        f.write("XML-Item-" + str(x) + " " + idList[x] + "\n")
        f.write(the_XML + "\n")
    x += 1
f.close

GARSSParsev001.py

# GARSSParsev001.py 
# By Adam Crymble (acrymbl@uwo.ca) as part of the Network in Canadian History & Environment's "Dashboard" project
#!/usr/bin/python

f = open('NiCHE-GA-RSS.txt', 'r')
the_XML = f.read()
f.close

while the_XML.find("XML-Item-01") != -1:
    one_XML = the_XML.find("XML-Item-01")
    the_XML = the_XML[0:one_XML]

PointsList = []
for i in range(0,31):
    PointEnd = the_XML.find("</Value>")
    PointStart = the_XML.find("<Value>")
    PointOnly = the_XML[PointStart+7: PointEnd]
    the_XML = the_XML[PointEnd  + 7:]
    PointsList.append(PointOnly)
print PointsList

Week1 = 0
Week2 = 0
Week3 = 0
Week4 = 0

#not adding properly
i = 30
while i > -1:
    if i > 23:
        print "Adding " + PointsList[i] + " to Week 1 i equals " + str(i)
        Week1 = Week1 + int(PointsList[i])
    elif i < 24 and i > 16:
        print "Adding " + PointsList[i] + " to Week 2 i equals " + str(i)
        Week2 = Week2 + int(PointsList[i])
    elif i > 9 and i < 17:
        print "Adding " + PointsList[i] + " to Week 3 i equals " + str(i)
        Week3 = Week3 + int(PointsList[i])
    elif i > 2 and i < 10:
        print "Adding " + PointsList[i] + " to Week 4 i equals " + str(i)
        Week4 = Week4 + int(PointsList[i])
    i -= 1
print "Week 1 is " + str(Week1)
print "Week 2 is " + str(Week2)
print "Week 3 is " + str(Week3)
print "Week 4 is " + str(Week4)
traffic = [Week1, Week2, Week3, Week4]
print traffic

RSStoLCDv003.py

# RSStoLCDv002.py By Adam Crymble, 2009. 
# Adapted from original by Adam Stelmack, Phidgets.
#!/usr/bin/env python

__author__ = 'Adam Crymble'
__version__ = '0.0.3'
__date__ = 'April 15 2009'

#Basic imports
from ctypes import *
import sys
from time import sleep
import random
#Phidget specific imports
from Phidgets.PhidgetException import *
from Phidgets.Events.Events import *
from Phidgets.Devices.TextLCD import TextLCD

# module Imports
# NiCHERSSv001 gets Article Titles; GARSSParsev001 gets Visitor Stats
import NiCHERSSv001
import GARSSParsev001

#Create an TextLCD object
textLCD = TextLCD()

#Information Display Function
def DisplayDeviceInfo():
    try:
        isAttached = textLCD.isAttached()
        type = textLCD.getDeviceType()
        serialNo = textLCD.getSerialNum()
        version = textLCD.getDeviceVersion()
        rowCount = textLCD.getRowCount()
        columnCount = textLCD.getColumnCount()
    except PhidgetException, e:
        print "Phidget Exception %i: %s" % (e.code, e.message)
        return 1
    print "|------------|----------------------------------|--------------|------------|"
    print "|- Attached -|-              Type              -|- Serial No. -|-  Version -|"
    print "|------------|----------------------------------|--------------|------------|"
    print "|- %8s -|- %30s -|- %10d -|- %8d -|" % (isAttached, type, serialNo, version)
    print "|------------|----------------------------------|--------------|------------|"
    print "Number of Rows: %i -- Number of Columns: %i" % (rowCount, columnCount)
    return 0

#Event Handler Callback Functions
def TextLCDAttached(e):
    attached = e.device
    print "TextLCD %i Attached!" % (attached.getSerialNum())
    return 0

def TextLCDDetached(e):
    detached = e.device
    print "TextLCD %i Detached!" % (detached.getSerialNum())
    return 0

def TextLCDError(e):
    print "Phidget Error %i: %s" % (e.eCode, e.description)
    return 0

#Main Program Code
try:
    textLCD.setOnAttachHandler(TextLCDAttached)
    textLCD.setOnDetachHandler(TextLCDDetached)
    textLCD.setOnErrorhandler(TextLCDError)
except PhidgetException, e:
    print "Phidget Exception %i: %s" % (e.code, e.message)
    print "Exiting...."
    exit(1)

print "Opening phidget object...."

try:
    textLCD.openPhidget()
except PhidgetException, e:
    print "Phidget Exception %i: %s" % (e.code, e.message)
    print "Exiting...."
    exit(1)

print "Waiting for attach...."

try:
    textLCD.waitForAttach(10000)
except PhidgetException, e:
    print "Phidget Exception %i: %s" % (e.code, e.message)
    try:
        textLCD.closePhidget()
    except PhidgetException, e:
        print "Phidget Exception %i: %s" % (e.code, e.message)
        print "Exiting...."
        exit(1)
    print "Exiting...."
    exit(1)
else:
    DisplayDeviceInfo()


itemCount = 0

#output starts here.
#Need to set up a timer that will run NiCHERSS.py every hour or so to update the list.
#must remove the &amp &quot characters when first saved to the List. Accented characters dont' work.
#need a restart button that stops the program then calls this module to play again from the start.

try:
    print NiCHERSSv001.itemList
    print NiCHERSSv001.itemCount
    print GARSSParsev001.traffic
    while 1 == 1:
        textLCD.setDisplayString(0, "")
        textLCD.setDisplayString(1, "")
        choose1 = random.randrange(0, NiCHERSSv001.itemCount + 4, 1)

        #Displays Article titles
        if choose1 < (NiCHERSSv001.itemCount):
            itemLen = len(NiCHERSSv001.itemList[choose1])
            tickerTracker = 0
            if itemLen < 20:
                textLCD.setDisplayString(0, "Recent Article:")
                textLCD.setDisplayString(1, NiCHERSSv001.itemList[choose1][0:20])
                sleep(3)
            else:
                #if the current letter is less than 19 letters from the end
                while itemLen - 19 > tickerTracker:
                    if tickerTracker == 0:
                        textLCD.setDisplayString(0, "Recent Article:")
                        textLCD.setDisplayString(1, NiCHERSSv001.itemList[choose1][0:20])
                        tickerTracker += 1
                        sleep(2)
                    elif tickerTracker < itemLen-18:
                        textLCD.setDisplayString(0, "Recent Article:")
                        textLCD.setDisplayString(1, NiCHERSSv001.itemList[choose1][tickerTracker:tickerTracker+20])
                        tickerTracker += 1
                        sleep(0.2)
                    else:
                        textLCD.setDisplayString(0, "Recent Article:")
                        textLCD.setDisplayString(1, NiCHERSSv001.itemList[choose1][tickerTracker:tickerTracker+20])
                        tickerTracker = 0
        #Displays Visitor Stats
        else:
            print choose1
            choose1 = choose1 - NiCHERSSv001.itemCount
            if choose1 == 1:
                textLCD.setDisplayString(0, "P-Nord Visitors:")
                textLCD.setDisplayString(1, "this Week: " + str(GARSSParsev001.traffic[choose1]))
                sleep(3)
            else:
                print GARSSParsev001.traffic[choose1]
                textLCD.setDisplayString(0, "P-Nord Visitors:")
                textLCD.setDisplayString(1, str(choose1 + 1) + " weeks ago: " + str(GARSSParsev001.traffic[choose1]))
                sleep(3)
        sleep(3)
    exit(0)
except PhidgetException, e:
    print "Phidget Exception %i: %s" % (e.code, e.message)
    print "Exiting...."
    exit(1)


print "Press Enter to quit...."

chr = sys.stdin.read(1)

print "Closing..."

textLCD.setDisplayString(0, "")
textLCD.setDisplayString(1, "")

try:
    textLCD.closePhidget()
except PhidgetException, e:
    print "Phidget Exception %i: %s" % (e.code, e.message)
    print "Exiting...."
    exit(1)

print "Done."
exit(0)