Crammer-Singer Multi-Class SVMΒΆ

Comparing different solvers on a standard multi-class SVM problem.

Python source code: multi_class_svm.py

from time import time
import numpy as np

#from sklearn.datasets import fetch_mldata
from sklearn.datasets import load_digits
from sklearn.cross_validation import train_test_split
#from sklearn.svm import LinearSVC

from pystruct.models import CrammerSingerSVMModel
from pystruct.learners import (NSlackSSVM, OneSlackSSVM,
                               SubgradientSSVM)

# do a binary digit classification
#digits = fetch_mldata("MNIST original")
digits = load_digits()
X, y = digits.data, digits.target
#X = X / 255.
X = X / 16.
y = y.astype(np.int)
X_train, X_test, y_train, y_test = train_test_split(X, y)


# we add a constant 1 feature for the bias
X_train_bias = np.hstack([X_train, np.ones((X_train.shape[0], 1))])
X_test_bias = np.hstack([X_test, np.ones((X_test.shape[0], 1))])

pbl = CrammerSingerSVMModel(n_features=X_train_bias.shape[1], n_classes=10)
n_slack_svm = NSlackSSVM(pbl, verbose=0, check_constraints=False, C=20,
                         batch_size=-1, tol=1e-2)
one_slack_svm = OneSlackSSVM(pbl, verbose=50, check_constraints=False, C=.20,
                             max_iter=10000, tol=.001, show_loss_every=10)
subgradient_svm = SubgradientSSVM(pbl, C=20, learning_rate=0.000001,
                                  max_iter=1000, verbose=0)

# n-slack cutting plane ssvm
#start = time()
#n_slack_svm.fit(X_train_bias, y_train)
#time_n_slack_svm = time() - start
#y_pred = np.hstack(n_slack_svm.predict(X_test_bias))
#print("Score with pystruct n-slack ssvm: %f (took %f seconds)"
      #% (np.mean(y_pred == y_test), time_n_slack_svm))

## 1-slack cutting plane ssvm
start = time()
one_slack_svm.fit(X_train_bias, y_train)
time_one_slack_svm = time() - start
y_pred = np.hstack(one_slack_svm.predict(X_test_bias))
print("Score with pystruct 1-slack ssvm: %f (took %f seconds)"
      % (np.mean(y_pred == y_test), time_one_slack_svm))

#online subgradient ssvm
#start = time()
#subgradient_svm.fit(X_train_bias, y_train)
#time_subgradient_svm = time() - start
#y_pred = np.hstack(subgradient_svm.predict(X_test_bias))

#print("Score with pystruct subgradient ssvm: %f (took %f seconds)"
      #% (np.mean(y_pred == y_test), time_subgradient_svm))

# because of the way I construct psi, we use half the C
# the standard one-vs-rest multi-class would probably be as good and faster
# but solving a different model
#libsvm = LinearSVC(multi_class='crammer_singer', C=10)
#start = time()
#libsvm.fit(X_train, y_train)
#time_libsvm = time() - start
#print("Score with sklearn and libsvm: %f (took %f seconds)"
      #% (libsvm.score(X_test, y_test), time_libsvm))

#from IPython.core.debugger import Tracer
#Tracer()()