del.icio.us .:.
For my SeedCamp talk I've built a small application using Yahoo! Query Language and oAuth. To try it out, enter the stock ticker symbol of your choice. (YHOO, MSFT, GOOG, and AAPL are a few of my favorites.)
A short while back, the smart folks behind Yahoo! Pipes released the next iteration, Yahoo! Query Language. YQL treats everything on the Web like a database table; the syntax will be familiar to anyone who's ever used MySQL. Here's how we're getting the data for the demo above:
select * from html where url="http://finance.yahoo.com/q?s=yhoo" and xpath='//div[@id="yfi_headlines"]/div[2]/ul/li'
This does exactly what you think it does: it requests the Yahoo! Finance listing for the ticker of your choice (YHOO, in this case) and uses XPath to grab just the recent headlines. Go try out the YQL console now, if you want; this example is all the way down the list of Available Data Tables, on the right of the page.
There is, of course, one small complication: as of right now--they tell me this will change soon for public data--all YQL calls need to be signed with oAuth. As long as you are absolutely sure you're not revealing a consumer key that is also used to access user data for some other application, a two-legged oAuth call can be done entirely on the client side with JavaScript. Here's how I did it:
( function() {
var trueName = '';
for (var i = 0; i < 16; i++) { trueName += String.fromCharCode(Math.floor(Math.random() * 26) + 97); }
window[trueName] = {};
var $ = window[trueName];
$.d = document;
$.b = $.d.getElementsByTagName('BODY')[0];
$.f = function() {
return {
init : function(target) {
$.s = $.d.getElementById(target);
$.s.q = $.d.createElement('INPUT');
$.s.q.onchange = function() {
$.f.runQuery();
};
$.s.appendChild($.s.q);
$.s.r = $.d.createElement('UL');
$.s.appendChild($.s.r);
},
runFunction : [],
runQuery : function() {
var n = $.f.runFunction.length;
var id = trueName + '.f.runFunction[' + n + ']';
$.f.runFunction[n] = function(r) {
$.f.renderQuery(r);
$.f.removeScript(id);
};
var consumerKey = '--consumer key goes here--';
var sharedSecret = '--shared secret goes here--';
var apiEndpoint = 'http://query.yahooapis.com/v1/yql?callback=' + id + '&format=json';
var apiQuery = 'select * from html where url="http://finance.yahoo.com/q?s=' + $.s.q.value + '" and xpath=\'//div[@id="yfi_headlines"]/div[2]/ul/li\'';
var url = $.f.oAuthRequest(consumerKey, sharedSecret, apiEndpoint, apiQuery);
$.f.runScript(url, id);
},
renderQuery : function(r) {
$.s.r.innerHTML = '';
if (r.error) {
var err = $.d.createElement('LI');
err.innerHTML = "Error:" + r.error.description;
$.s.r.appendChild(err);
} else {
if (typeof r.query.results.li !== 'undefined') {
for (var i = 0; i < r.query.results.li.length; i++) {
var e = r.query.results.li[i];
var li = $.d.createElement('LI');
var a = $.d.createElement('A');
a.innerHTML = e.a.content;
a.href = e.a.href;
a.target = '_blank';
li.appendChild(a);
var cite = $.d.createElement('CITE');
cite.innerHTML = ' - ' + e.cite.content + ' ' + e.cite.span;
li.appendChild(cite);
$.s.r.appendChild(li);
}
} else {
var notFound = $.d.createElement('LI');
notFound.innerHTML = "Sorry, headlines for " + $.s.q.value + " not found.";
$.s.r.appendChild(notFound);
}
}
},
runScript : function(url, id) {
var s = $.d.createElement('script');
s.id = id;
s.type ='text/javascript';
s.charset = 'utf-8';
s.src = url;
$.b.appendChild(s);
},
removeScript : function(id) {
var s = $.d.getElementById(id);
if (s !== null && s.tagName === 'SCRIPT') {
s.parentNode.removeChild(s);
}
},
percentEscape : function(r) {
// does normal encodeURIComponent and then takes care of !, *, ', (, and )
return encodeURIComponent(r).replace("!","%21","g").replace("*","%2A","g").replace("'","%27","g").replace("(","%28","g").replace(")","%29","g");
},
oAuthRequest : function(key, secret, apiEndpoint, apiQuery) {
// percent-escape our query
var encodedQuery = $.f.percentEscape(apiQuery);
// yes, this winds up getting done twice, for the actual query only
// get our timestamp
var timestamp = Math.floor(new Date().getTime()/1000);
// make a random nonce
var nonce = '';
for (var i = 0; i < 10; i++) { nonce += String.fromCharCode(Math.floor(Math.random() * 26) + 97); }
// the URL we're querying. Example: http://query.yahooapis.com/v1/yql
var theUrl = apiEndpoint.split('?')[0];
// non-query-related args and values (callback=foo&format=json, in our case)
// note: the leading ? has been REMOVED
var theHead = apiEndpoint.split('?')[1];
// the oAuth goodies
var theBody = '';
// IMPORTANT! we do NOT want to show our consumer key for anything requiring user login!
theBody += '&oauth_consumer_key=' + key;
theBody += '&oauth_nonce=' + nonce;
theBody += '&oauth_signature_method=HMAC-SHA1';
theBody += '&oauth_timestamp=' + timestamp;
theBody += '&oauth_version=1.0';
// add an unencoded &q= to percent-escaped query:
theBody += '&q=' + encodedQuery;
// percent-escape theUrl, theHead, and theBody
var theData = "GET" + "&" + $.f.percentEscape(theUrl) + '&' + $.f.percentEscape(theHead) + $.f.percentEscape(theBody);
// did you notice? the &q= from above was percent-escaped with theBody
// get the base-64 hashed message authentication code (HMAC) using SHA-1 and our shared secret
var theSig = $.f.b64_hmac_sha1(secret + '&', theData);
// hand it back, ready to submit
var signedUrl = apiEndpoint + theBody + '&oauth_signature=' + $.f.percentEscape(theSig);
return (signedUrl);
// did you notice? All args except for oauth_signature are in alphabetical order
},
b64_hmac_sha1 : function(k,d,_p,_z){
if(!_p){_p='=';}if(!_z){_z=8;}function _f(t,b,c,d){if(t<20){return(b&c)|((~b)&d);}if(t<40){return b^c^d;}if(t<60){return(b&c)|(b&d)|(c&d);}return b^c^d;}function _k(t){return(t<20)?1518500249:(t<40)?1859775393:(t<60)?-1894007588:-899497514;}function _s(x,y){var l=(x&0xFFFF)+(y&0xFFFF),m=(x>>16)+(y>>16)+(l>>16);return(m<<16)|(l&0xFFFF);}function _r(n,c){return(n<<c)|(n>>>(32-c));}function _c(x,l){x[l>>5]|=0x80<<(24-l%32);x[((l+64>>9)<<4)+15]=l;var w=[80],a=1732584193,b=-271733879,c=-1732584194,d=271733878,e=-1009589776;for(var i=0;i<x.length;i+=16){var o=a,p=b,q=c,r=d,s=e;for(var j=0;j<80;j++){if(j<16){w[j]=x[i+j];}else{w[j]=_r(w[j-3]^w[j-8]^w[j-14]^w[j-16],1);}var t=_s(_s(_r(a,5),_f(j,b,c,d)),_s(_s(e,w[j]),_k(j)));e=d;d=c;c=_r(b,30);b=a;a=t;}a=_s(a,o);b=_s(b,p);c=_s(c,q);d=_s(d,r);e=_s(e,s);}return[a,b,c,d,e];}function _b(s){var b=[],m=(1<<_z)-1;for(var i=0;i<s.length*_z;i+=_z){b[i>>5]|=(s.charCodeAt(i/8)&m)<<(32-_z-i%32);}return b;}function _h(k,d){var b=_b(k);if(b.length>16){b=_c(b,k.length*_z);}var p=[16],o=[16];for(var i=0;i<16;i++){p[i]=b[i]^0x36363636;o[i]=b[i]^0x5C5C5C5C;}var h=_c(p.concat(_b(d)),512+d.length*_z);return _c(o.concat(h),512+160);}function _n(b){var t="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/",s='';for(var i=0;i<b.length*4;i+=3){var r=(((b[i>>2]>>8*(3-i%4))&0xFF)<<16)|(((b[i+1>>2]>>8*(3-(i+1)%4))&0xFF)<<8)|((b[i+2>>2]>>8*(3-(i+2)%4))&0xFF);for(var j=0;j<4;j++){if(i*8+j*6>b.length*32){s+=_p;}else{s+=t.charAt((r>>6*(3-j))&0x3F);}}}return s;}function _x(k,d){return _n(_h(k,d));}return _x(k,d);
// heavily optimized and compressed version of http://pajhome.org.uk/crypt/md5/sha1.js
// _p = b64pad, _z = charcter size; not used here but I left them available just in case
}
};
}();
// substitute the div ID of your choice for "yql"
$.f.init('yql');
})();
Please go get your own oAuth application key at the YDN dashboard; using mine would be terribly bad karma. :)
I'm not going to go into a huge amount of detail here; I'm trying to figure out three-legged oAuth right now, and hopefully this approach won't be necessary in a little while. Still, it was fun to figure out client-side oAuth--it's quite a bit simpler than it looks in the Netflix library--and put something up.
Of particular interest here may be the compressed HMAC-SHA1 function; this was simplified from Paul Johnston's full-sized version, here: http://pajhome.org.uk/crypt/md5/sha1.js.
I think YQL is at least as significant of a development as Pipes. Pipes made client-side mash-ups possible; YQL will make possible a global API layer over anything you can touch with HTTP. More on this, later.
As always, have fun with this, and please let me know how it goes.
It also has an option to prevent the exposure of the Consumer Key and Secret by using a disposable AppId
Nobody has yet.