Posted By

giventocode on 09/28/14


Tagged

c#


Versions (?)

RssMLPrep


 / Published in: C#
 

URL: http://giventocode.com/build-a-recommendations-system-for-your-blog-or-web-site-using-azure-machine-learning-and-azure-mobile-services

Text preparation Azure Mobile Services scheduled job.

  1. using System.Threading.Tasks;
  2. using System.Web.Http;
  3. using Microsoft.WindowsAzure.Mobile.Service;
  4. using System.Net.Http;
  5. using Microsoft.WindowsAzure;
  6. using System;
  7. using System.IO;
  8. using System.Xml.XPath;
  9. using System.Text;
  10. using System.Security.Cryptography;
  11. using Microsoft.WindowsAzure.Storage.Table;
  12. using Microsoft.WindowsAzure.Storage;
  13. using System.Text.RegularExpressions;
  14. using System.Web;
  15.  
  16. namespace giventocode.m.ScheduledJobs
  17. {
  18. // A simple scheduled job which can be invoked manually by submitting an HTTP
  19. // POST request to the path "/jobs/sample".
  20.  
  21. public class RssMLPrep : ScheduledJob
  22. {
  23.  
  24. public async override Task ExecuteAsync()
  25. {
  26.  
  27. using (var httpClient = new HttpClient())
  28. {
  29.  
  30. var response = await httpClient.GetAsync(new Uri(CloudConfigurationManager.GetSetting("RssSource")));
  31. var data = await response.Content.ReadAsStringAsync();
  32. var xnodes = new XPathDocument(new MemoryStream(UTF8Encoding.UTF8.GetBytes(data)))
  33. .CreateNavigator()
  34. .Select("//item");
  35.  
  36. var table = GetCloudTable();
  37.  
  38.  
  39. foreach (XPathNavigator node in xnodes)
  40. {
  41. var title = node.SelectSingleNode("title").ToString();
  42.  
  43. //Include the title in the text and prepare text...
  44. var text = PrepareText(string.Format("{0} {1}", title, node.SelectSingleNode("description").ToString()));
  45.  
  46. //hash the title to remove invalid chars and use it as the key
  47. var rowKey = ComputeSH1Hash(title);
  48.  
  49. var tableOp = TableOperation.InsertOrMerge(new BlogEntry()
  50. {
  51. Text = text,
  52. PubDate = DateTime.Now,
  53. Link = node.SelectSingleNode("link").ToString(),
  54. RowKey = rowKey,
  55. PartitionKey = "BlogEntries",
  56. Title = title.Trim()
  57. });
  58.  
  59. await table.ExecuteAsync(tableOp);
  60.  
  61. }
  62.  
  63. }
  64. }
  65.  
  66.  
  67. private string ComputeSH1Hash(string data)
  68. {
  69. var retdata = string.Empty;
  70.  
  71. using (var sha = SHA1.Create())
  72. {
  73. var bytes = sha.ComputeHash(Encoding.UTF8.GetBytes(data));
  74. StringBuilder builder = new StringBuilder();
  75.  
  76. // Loop through each byte of the hashed data
  77. // and format each one as a hexadecimal string.
  78. for (int i = 0; i < bytes.Length; i++)
  79. {
  80. builder.Append(bytes[i].ToString("x2"));
  81. }
  82.  
  83. retdata = builder.ToString();
  84.  
  85. }
  86.  
  87. return retdata;
  88. }
  89.  
  90.  
  91. private CloudTable GetCloudTable()
  92. {
  93. var cloudStorageAccount = CloudStorageAccount.DevelopmentStorageAccount;
  94. CloudStorageAccount.TryParse(CloudConfigurationManager.GetSetting("StorageAcct"), out cloudStorageAccount);
  95. var tClient = cloudStorageAccount.CreateCloudTableClient();
  96. var _tref = tClient.GetTableReference(CloudConfigurationManager.GetSetting("SourceDataTable"));
  97.  
  98. _tref.CreateIfNotExists();
  99.  
  100. return _tref;
  101.  
  102. }
  103.  
  104. private string PrepareText(string htmlData)
  105. {
  106. //lower case all the data
  107. var data = htmlData.ToLower();
  108. data = data.Replace(System.Environment.NewLine, " ");
  109.  
  110. //remove CSS ...
  111. data = Regex.Replace(data, @"<style[^><]*>.*</style>", string.Empty, RegexOptions.Singleline);
  112.  
  113. //remove HTML ...
  114. data = Regex.Replace(data, "<.*?>", string.Empty);
  115. data = HttpUtility.HtmlDecode(data);
  116.  
  117. //Replace apostrophe with single quote ...
  118. data = data.Replace("’", "'");
  119.  
  120. //remove punctuation characters other than ' and -
  121. data = Regex.Replace(data, @"[^\w\'\s\-]", " ");
  122.  
  123. //stop words...
  124. var sws = new string[] { "a's", "able", "about", "above", "according", "accordingly", "across", "actually", "after", "afterwards", "again", "against", "ain't", "all", "allow", "allows", "almost", "alone", "along", "already", "also", "although", "always", "am", "among", "amongst", "an", "and", "another", "any", "anybody", "anyhow", "anyone", "anything", "anyway", "anyways", "anywhere", "apart", "appear", "appreciate", "appropriate", "are", "aren't", "around", "as", "aside", "ask", "asking", "associated", "at", "available", "away", "awfully", "be", "became", "because", "become", "becomes", "becoming", "been", "before", "beforehand", "behind", "being", "believe", "below", "beside", "besides", "best", "better", "between", "beyond", "both", "brief", "but", "by", "c'mon", "c's", "came", "can", "can't", "cannot", "cant", "cause", "causes", "certain", "certainly", "changes", "clearly", "co", "com", "come", "comes", "concerning", "consequently", "consider", "considering", "contain", "containing", "contains", "corresponding", "could", "couldn't", "course", "currently", "definitely", "described", "despite", "did", "didn't", "different", "do", "does", "doesn't", "doing", "don't", "done", "down", "downwards", "during", "each", "edu", "eg", "eight", "either", "else", "elsewhere", "enough", "entirely", "especially", "et", "etc", "even", "ever", "every", "everybody", "everyone", "everything", "everywhere", "ex", "exactly", "example", "except", "far", "few", "fifth", "first", "five", "followed", "following", "follows", "for", "former", "formerly", "forth", "four", "from", "further", "furthermore", "get", "gets", "getting", "given", "gives", "go", "goes", "going", "gone", "got", "gotten", "greetings", "had", "hadn't", "happens", "hardly", "has", "hasn't", "have", "haven't", "having", "he", "he's", "hello", "help", "hence", "her", "here", "here's", "hereafter", "hereby", "herein", "hereupon", "hers", "herself", "hi", "him", "himself", "his", "hither", "hopefully", "how", "howbeit", "however", "i'd", "i'll", "i'm", "i've", "ie", "if", "ignored", "immediate", "in", "inasmuch", "inc", "indeed", "indicate", "indicated", "indicates", "inner", "insofar", "instead", "into", "inward", "is", "isn't", "it", "it'd", "it'll", "it's", "its", "itself", "just", "keep", "keeps", "kept", "know", "known", "knows", "last", "lately", "later", "latter", "latterly", "least", "less", "lest", "let", "let's", "like", "liked", "likely", "little", "look", "looking", "looks", "ltd", "mainly", "many", "may", "maybe", "me", "mean", "meanwhile", "merely", "might", "more", "moreover", "most", "mostly", "much", "must", "my", "myself", "name", "namely", "nd", "near", "nearly", "necessary", "need", "needs", "neither", "never", "nevertheless", "new", "next", "nine", "no", "nobody", "non", "none", "noone", "nor", "normally", "not", "nothing", "novel", "now", "nowhere", "obviously", "of", "off", "often", "oh", "ok", "okay", "old", "on", "once", "one", "ones", "only", "onto", "or", "other", "others", "otherwise", "ought", "our", "ours", "ourselves", "out", "outside", "over", "overall", "own", "particular", "particularly", "per", "perhaps", "placed", "please", "plus", "possible", "presumably", "probably", "provides", "que", "quite", "qv", "rather", "rd", "re", "really", "reasonably", "regarding", "regardless", "regards", "relatively", "respectively", "right", "said", "same", "saw", "say", "saying", "says", "second", "secondly", "see", "seeing", "seem", "seemed", "seeming", "seems", "seen", "self", "selves", "sensible", "sent", "serious", "seriously", "seven", "several", "shall", "she", "should", "shouldn't", "since", "six", "so", "some", "somebody", "somehow", "someone", "something", "sometime", "sometimes", "somewhat", "somewhere", "soon", "sorry", "specified", "specify", "specifying", "still", "sub", "such", "sup", "sure", "t's", "take", "taken", "tell", "tends", "th", "than", "thank", "thanks", "thanx", "that", "that's", "thats", "the", "their", "theirs", "them", "themselves", "then", "thence", "there", "there's", "thereafter", "thereby", "therefore", "therein", "theres", "thereupon", "these", "they", "they'd", "they'll", "they're", "they've", "think", "third", "this", "thorough", "thoroughly", "those", "though", "three", "through", "throughout", "thru", "thus", "to", "together", "too", "took", "toward", "towards", "tried", "tries", "truly", "try", "trying", "twice", "two", "un", "under", "unfortunately", "unless", "unlikely", "until", "unto", "up", "upon", "us", "use", "used", "useful", "uses", "using", "usually", "value", "various", "very", "via", "viz", "vs", "want", "wants", "was", "wasn't", "way", "we", "we'd", "we'll", "we're", "we've", "welcome", "well", "went", "were", "weren't", "what", "what's", "whatever", "when", "whence", "whenever", "where", "where's", "whereafter", "whereas", "whereby", "wherein", "whereupon", "wherever", "whether", "which", "while", "whither", "who", "who's", "whoever", "whole", "whom", "whose", "why", "will", "willing", "wish", "with", "within", "without", "won't", "wonder", "would", "wouldn't", "yes", "yet", "you", "you'd", "you'll", "you're", "you've", "your", "yours", "yourself", "yourselves", "zero" };
  125.  
  126. foreach (var w in sws)
  127. {
  128. data = data.Replace(string.Format(" {0} ", w), " ");
  129. }
  130.  
  131. data = data.Replace(" ", " ");
  132.  
  133. return data;
  134. }
  135.  
  136. }
  137. public class BlogEntry : TableEntity
  138. {
  139.  
  140. public string Text { get; set; }
  141. public DateTime PubDate { get; set; }
  142.  
  143. public string Link { get; set; }
  144. public string Title { get; set; }
  145. }
  146.  
  147.  
  148. }

Report this snippet  

You need to login to post a comment.