PRML notes

Author: ifding

Diff for: PRML/ch1.pdf

@@ -28,8 +28,12 @@
     - [DDoS](./security/ddos/README.md)
     - [Security Cheatsheets](./security-cheatsheets)
     - [IoT](./security/iot-security.md)
+
+## Machine Learning
 - [Machine Learning](./machine-learning/README.md)
 - [Maths](./maths/README.md)
+- Pattern_Recognition_And_Machine_Learning
+    - [Ch1 Introduction](./PRML/ch1.pdf)
 
 
 ## Resources

Diff for: PRML/ch1/figure/decision_boundary.png

@@ -15,6 +15,8 @@
 * Backtracking
 
 ### 3. Sorting
+* [sort and scan](sort_and_scan)
+* [inverse quicksort](adversarial_input)
 
 ### 4. Binary Search
 
@@ -27,5 +29,5 @@
 * Greedy algorithm
 
 ### 7. PageRank
-* [PageRank and Map-Reduce Implement (Chinese)](http://www.cnblogs.com/fengfenggirl/p/pagerank-introduction.html)
+* [mini web search engine](mini_web_search_engine)
 

Diff for: PRML/ch1/figure/gaussian.png

@@ -0,0 +1,113 @@
+#include <iostream>
+#include <fstream>
+#include <cstdlib>
+#include <algorithm>
+#include <assert.h>
+
+using namespace std;
+
+struct Node {
+  int key;
+  int size;
+  Node *left;
+  Node *right;
+  Node (int k) { key = k; size = 1; left = right = NULL; }
+};
+
+void fix_size(Node *T)
+{
+  T->size = 1;
+  if (T->left) T->size += T->left->size;
+  if (T->right) T->size += T->right->size;
+}
+
+// prints out the inorder traversal of T (i.e., the contents of T in sorted order)
+void print_inorder(Node *T)
+{
+  if (T == NULL) return;
+  print_inorder(T->left);
+  cout << T->key << "\n";
+  print_inorder(T->right);
+}
+
+// Split tree T on rank r into tree L (containing ranks < r) and 
+// a tree R (containing ranks >= r)
+void split(Node *T, int r, Node **L, Node **R)
+{
+  if (T == NULL) {
+    *L = NULL;
+    *R = NULL;
+    return;
+  }
+  int rank_of_root = T->left ? T->left->size : 0;
+  if (r <= rank_of_root) {
+    // recursively split left subtree
+    split(T->left, r, L, &T->left);
+    *R = T;
+  } else {
+    split(T->right, r-rank_of_root-1, &T->right, R);
+    *L = T;
+  }
+  fix_size(T);
+}
+
+// insert value v at rank r
+Node *insert_random(Node *T, int v, int r)
+{
+  // If k is the Nth node inserted into T, then:
+  // with probability 1/N, insert k at the root of T
+  // otherwise, insert_random k recursively left or right of the root of T
+  if (T == NULL) return new Node(v);
+  if (rand() % (T->size + 1) == 0) {
+    // insert at root
+    Node *new_root = new Node(v);
+    split(T, r, &new_root->left, &new_root->right);
+    fix_size(new_root);
+    return new_root;
+  }
+  int rank_of_root = T->left ? T->left->size : 0;
+  if (r <= rank_of_root) T->left = insert_random(T->left, v, r);
+  else T->right = insert_random(T->right, v, r - rank_of_root - 1);
+  fix_size(T);
+  return T;
+}
+
+Node *T;
+
+void generate(int N)
+{
+  if (N==1) {
+    T = insert_random(T, 1, 0);
+    return;
+  }
+  generate(N-1);
+  int pivot = 123456789%N;
+
+  //  Old array-based solution:
+  //  for (int i=N-1; i>pivot; i--)
+  //    A[i] = A[i-1];
+  //  A[pivot] = N;
+
+  // Faster solution by encoding sequence in a balanced BST:
+  T = insert_random(T, N, pivot);
+}
+
+int main(int argc, char *argv[])
+{
+  if (argc != 2) {
+    cout << "Usage: bad1 <input size>\n";
+    return 0;
+  }
+
+  int N = atoi(argv[1]);  // get first command-line argument
+  if (N<1 || N>100000) {
+    cout << "Invalid input size!\n";
+    return 0;
+  }
+  
+  cout << N << "\n";
+  generate(N);
+  print_inorder(T);
+  
+  return 0;
+}

Diff for: PRML/ch1/figure/mle_bias.png

@@ -0,0 +1,122 @@
+#include <iostream>
+#include <algorithm>
+using namespace std;
+
+// Generate a random number in the range 0..N-1
+int get_rand(int N)
+{
+  // I forgot to attend the lecture where Prof Dean. explained how to generate
+  // random numbers well, so hopefully this will work ok...
+  // It looks like get_rand(N) always returns the same number if you call
+  // it multiple times with the same value of N, but how could someone 
+  // possibly exploit that to make quicksort run slowly...
+  return 123456789 % N;
+}
+
+struct Node {
+  int key;
+  int size;
+  Node *left;
+  Node *right;
+  Node (int k) { key = k; size = 1; left = right = NULL; }
+};
+
+void fix_size(Node *T)
+{
+  T->size = 1;
+  if (T->left) T->size += T->left->size;
+  if (T->right) T->size += T->right->size;
+}
+
+// prints out the inorder traversal of T (i.e., the contents of T in sorted order)
+void print_inorder(Node *T)
+{
+  if (T == NULL) return;
+  print_inorder(T->left);
+  cout << T->key << " ";
+  print_inorder(T->right);
+}
+
+// Split tree T on rank r into tree L (containing ranks < r) and 
+// a tree R (containing ranks >= r)
+void split(Node *T, int r, Node **L, Node **R)
+{
+  // TBD: please fill in this function appropriately
+  if (T==NULL) {
+    *L=NULL;
+    *R=NULL;
+    return;
+  }
+  int rank_of_root = T->left ? T->left->size : 0;
+  if (r>rank_of_root) {
+    split(T->right, r - rank_of_root - 1, &T->right, R);
+    *L = T;
+  } else {
+    split(T->left, r, L, &T->left);
+    *R = T;
+  }
+  fix_size(T);
+}
+
+// insert value v at rank r
+Node *insert_keep_balanced(Node *T, int v, int r)
+{
+  // TBD: fill this in
+  if (T==NULL) return new Node(v);
+  if (rand() % (T->size + 1) == 0) {
+    Node *new_root = new Node(v);
+    split(T, r, &new_root->left, &new_root->right);
+    fix_size(new_root);
+    return new_root; 
+  }
+  int rank_of_root = T->left ? T->left->size : 0;
+  if (r > rank_of_root) T->right = insert_keep_balanced(T->right, v, r - rank_of_root - 1);
+  else T->left = insert_keep_balanced(T->left, v, r);
+  fix_size(T);
+  return T;
+}
+
+// implement in a balanced BST
+void generate_adversarial(Node **T, int N)
+{
+  if (N==1) {
+    *T = insert_keep_balanced(*T, 1, 0);
+    return;
+  }
+  generate_adversarial(T, N-1);
+  int pivot = get_rand(N);
+
+  *T = insert_keep_balanced(*T, N, pivot);
+}
+
+
+int main(int argc, char *argv[])
+{
+  if (argc != 2) {
+    cout << "Usage: bad1 <input size>\n";
+    return 0;
+  }
+
+  int N = atoi(argv[1]);  // get first command-line argument
+  if (N<1 || N>100000) {
+    cout << "Invalid input size!\n";
+    return 0;
+  }
+ 
+  // Generate and print bad input of size N for prog1
+  // (currently just generates an input of N random numbers)
+  cout << N << "\n";
+  
+  //int *A = new int[N];
+  //for (int i=0; i<N; i++)
+  //  A[i] = rand() % 1000000;
+
+  Node *T=NULL;
+  generate_adversarial(&T, N);
+  
+  print_inorder(T);
+  //for (int i=0; i<N; i++)
+  //  cout << A[i] << "\n";
+  
+  return 0;
+}

Diff for: PRML/ch1/figure/probability_density.png

@@ -0,0 +1,48 @@
+#include <iostream>
+#include <algorithm>
+using namespace std;
+
+// Solve the "longest line of sight" problem from lecture...
+
+int main(void)
+{
+  int N; // size of input
+  if (!(cin >> N) || N < 1 || N > 100000) {
+    cout << "Invalid input size!\n";
+    return 0;
+  }
+
+  // read input
+  int *A = new int[N];
+  for (int i=0; i<N; i++) 
+    if (!(cin >> A[i]) || A[i]<0) {
+      cout << "Invalid input!\n";
+      return 0;
+    }
+  
+  // compute max to left and max to right from each point
+  int *max_to_left = new int[N];
+  int *max_to_right = new int[N];
+  max_to_left[1] = A[0];
+  for (int i=2; i<N; i++) 
+    max_to_left[i] = max(A[i-1], max_to_left[i-1]);
+  max_to_right[N-2] = A[N-1];
+  for (int i=N-3; i>=0; i--) 
+    max_to_right[i] = max(A[i+1], max_to_right[i+1]);
+
+  // now find the longest line of sight:
+  // for each element, scan left and right until we reach the
+  // first larger element; record farthest such element found.
+  int best = -1;
+  for (int i=0; i<N; i++) {
+    if (i>0 && max_to_left[i] >= A[i]) // is there someone larger on left?
+      for (int j=i-1; j>=0; j--) // scan left until we reach someone larger
+	if (A[j] >= A[i]) { best = max(best, i-j); break; }	
+    if (i<N-1 && max_to_right[i] >= A[i]) // is there someone larger on right?
+      for (int j=i+1; j<N; j++) // scan right until we reach someone larger
+	if (A[j] >= A[i]) { best = max(best, j-i); break; }
+  }
+  
+  cout << "Longest line of sight: " << best << "\n";
+  return 0;
+}

Diff for: PRML/ch1/main.tex

@@ -0,0 +1,67 @@
+#include <iostream>
+#include <algorithm>
+using namespace std;
+
+// Find number of distinct elements using a hash table
+
+struct Node {
+  int val;
+  Node *next;
+  Node (int v, Node *n) { val = v; next = n; }
+};
+
+Node **T; // array of Node *s for hash table
+int table_size = 10000;  
+
+// check out this awesome hash function!
+int hash(int x)
+{
+  return ((unsigned int)x * 3 + 17) % table_size;
+}
+
+bool find(int x)
+{
+  for (Node *n = T[hash(x)]; n != NULL; n = n->next)
+    if (n->val == x) return true;
+  return false;
+}
+
+void insert(int x)
+{
+  int index = hash(x);
+  T[index] = new Node(x, T[index]);
+}
+
+int main(void)
+{
+  int N; // size of input
+  if (!(cin >> N) || N < 1 || N > 100000) {
+    cout << "Invalid input size!\n";
+    return 0;
+  }
+
+  // read input
+  int *A = new int[N];
+  for (int i=0; i<N; i++) 
+    if (!(cin >> A[i]) || A[i]<0) {
+      cout << "Invalid input!\n";
+      return 0;
+    }
+
+  // initialize hash table -- if I make it have size 10,000, then the
+  // average length of a linked list should be small...
+  T = new Node *[table_size]; 
+  for (int i=0; i<table_size; i++)
+    T[i] = NULL;
+
+  int count = 0;
+  for (int i=0; i<N; i++)
+    if (!find(A[i])) {
+      count++;
+      insert(A[i]);
+    }
+
+  cout << "The input contains " << count << " distinct integers.\n";
+
+  return 0;
+}