Introduction to Neural Networks for Java, Session 14

Session Material

I present here my solution for program 2. You can use the link below to download an Eclipse project for my solution.

[My Solution for Program 2]

Analyze.java Calculate the min/max for the inputs.

package com.heatonresearch.course.introneuralnet.program2;

import java.io.IOException;

import com.heatonresearch.book.introneuralnet.common.ReadCSV;

public class Analyze {
	
	public final static String[] COVER_TYPES = {
	    "Spruce/Fir", 			// 1
	    "Lodgepole Pine", 		// 2
	    "Ponderosa Pine", 		// 3
	    "Cottonwood/Willow", 	// 4
	    "Aspen", 				// 5
	    "Douglas-fir", 			// 6
	    "Krummholz" 			// 7
	};
	
	private double maxElevation;
	private double minElevation;
	private double maxAspect;
	private double minAspect;			
	private double maxSlope;
	private double minSlope;
	private double maxHWater;
	private double minHWater;
	private double maxVWater;
	private double minVWater;
	private double maxRoad;
	private double minRoad;
	private double maxShade9;
	private double minShade9;
	private double maxShade12;
	private double minShade12;
	private double maxShade3;
	private double minShade3;
	
	private int[] coverCount;
	private int count;
	
	public void analyze(String filename) throws IOException
	{
		this.count = 0;
		this.maxElevation = Double.MIN_VALUE;
		this.minElevation = Double.MAX_VALUE;
		this.maxAspect = Double.MIN_VALUE;
		this.minAspect = Double.MAX_VALUE;			
		this.maxSlope = Double.MIN_VALUE;
		this.minSlope = Double.MAX_VALUE;
		this.maxHWater = Double.MIN_VALUE;
		this.minHWater = Double.MAX_VALUE;
;		this.maxVWater = -1000;//Double.MIN_VALUE;
		this.minVWater = Double.MAX_VALUE;
		this.maxShade9 = Double.MIN_VALUE;
		this.minShade9 = Double.MAX_VALUE;
		this.maxShade12 = Double.MIN_VALUE;
		this.minShade12 = Double.MAX_VALUE;
		this.maxShade3 = Double.MIN_VALUE;
		this.minShade3 = Double.MAX_VALUE;
		this.maxRoad = Double.MIN_VALUE;
		this.minRoad = Double.MAX_VALUE;
		
		this.coverCount = new int[Analyze.COVER_TYPES.length];

		System.out.println("Please wait, performing initial analysis of data.");
		ReadCSV csv = new ReadCSV(filename,false);
		while( csv.next() )
		{			


				double elevation = Double.parseDouble(csv.get(0));
				double aspect = Double.parseDouble(csv.get(1));
				double slope = Double.parseDouble(csv.get(2));
				double hWater = Double.parseDouble(csv.get(3));
				double vWater = Double.parseDouble(csv.get(4));
				double road = Double.parseDouble(csv.get(5));
				double shade9 = Double.parseDouble(csv.get(6));
				double shade12 = Double.parseDouble(csv.get(7));
				double shade3 = Double.parseDouble(csv.get(8));
				int cover = Integer.parseInt(csv.get(54));
				maxElevation = Math.max(maxElevation, elevation);
				minElevation = Math.min(minElevation, elevation);
				maxAspect = Math.max(maxAspect, aspect);
				minAspect = Math.min(minAspect, aspect);
				maxSlope = Math.max(maxSlope, slope);
				minSlope = Math.min(minSlope, slope);
				maxHWater = Math.max(maxHWater, hWater);
				minHWater = Math.min(minHWater, hWater);
				maxVWater = Math.max(maxVWater, vWater);
				minVWater = Math.min(minVWater, vWater);
				maxRoad = Math.max(maxRoad, road);
				minRoad = Math.min(minRoad, road);
				maxShade9 = Math.max(maxShade9, shade9);
				minShade9 = Math.min(minShade9, shade9);
				maxShade12 = Math.max(maxShade12, shade12);
				minShade12 = Math.min(minShade12, shade12);
				maxShade3 = Math.max(maxShade3, shade3);
				minShade3 = Math.min(minShade3, shade3);
				this.coverCount[cover-1]++;
				count++;

		}
		
		System.out.println("Record count: " + count);
		System.out.println("Elevation, Max: " + maxElevation);
		System.out.println("Elevation, Min: " + minElevation);
		System.out.println("Aspect, Max: " + maxAspect);
		System.out.println("Aspect, Min: " + minAspect);
		System.out.println("Slope, Max: " + maxSlope);
		System.out.println("Slope, Min: " + minSlope);
		System.out.println("H. Water, Max: " + maxHWater);
		System.out.println("H. Water, Min: " + minHWater);
		System.out.println("V. Water, Max: " + maxVWater);
		System.out.println("V. Water, Min: " + minVWater);
		
		System.out.println("Cover breakdown:");
		for(int i=0;i<Analyze.COVER_TYPES.length;i++)
		{
			System.out.println(Analyze.COVER_TYPES[i] + ": " + this.coverCount[i]);
		}
	}

	public double getMaxElevation() {
		return maxElevation;
	}

	public double getMinElevation() {
		return minElevation;
	}

	public double getMaxAspect() {
		return maxAspect;
	}

	public double getMinAspect() {
		return minAspect;
	}

	public double getMaxSlope() {
		return maxSlope;
	}

	public double getMinSlope() {
		return minSlope;
	}

	public int[] getCoverCount() {
		return coverCount;
	}

	public int getCount() {
		return count;
	}

	public double getMaxHWater() {
		return maxHWater;
	}

	public double getMinHWater() {
		return minHWater;
	}

	public double getMaxVWater() {
		return maxVWater;
	}

	public double getMinVWater() {
		return minVWater;
	}

	public double getMaxRoad() {
		return maxRoad;
	}

	public double getMaxShade9() {
		return maxShade9;
	}

	public double getMaxShade12() {
		return maxShade12;
	}

	public double getMaxShade3() {
		return maxShade3;
	}

	public double getMinRoad() {
		return minRoad;
	}

	public double getMinShade9() {
		return minShade9;
	}

	public double getMinShade12() {
		return minShade12;
	}

	public double getMinShade3() {
		return minShade3;
	}
}

Config.java Config information for this program.

package com.heatonresearch.course.introneuralnet.program2;

public class Config {
	public static final String FILENAME = "c:\\data\\covtype.data";
	public static final int TRAINING_SIZE = 1000;
	public static final int MAX_HIDDEN_LAYER_1 = 20;
	public static final int MAX_HIDDEN_LAYER_2 = 20;
	public static final double LEARNING_RATE = 0.001;
	public static final double LEARNING_MOMENTUM = 0.3;
	public static final double TRUE = 1.0;
	public static final double FALSE = 0.0;
	public static final int EPOCHS = 500;
	public static final int COVER_SAMPLES = 2000;
}

Convert.java Convert the data to be fed into the neural network.

package com.heatonresearch.course.introneuralnet.program2;

import com.heatonresearch.book.introneuralnet.common.ReadCSV;

public class Convert {
	
	public static double percent(double value,double max,double min)
	{
		return (value-min)/(max-min);
	}
	
	public static void convert(Analyze analyze, ReadCSV line,double[] input,double[] ideal )
	{
		double elevation = Double.parseDouble(line.get(0));
		double aspect = Double.parseDouble(line.get(1));
		double slope = Double.parseDouble(line.get(2));
		double hWater = Double.parseDouble(line.get(3));
		double vWater = Double.parseDouble(line.get(4));
		double road = Double.parseDouble(line.get(5));
		double shade9 = Double.parseDouble(line.get(6));
		double shade12 = Double.parseDouble(line.get(7));
		double shade3 = Double.parseDouble(line.get(8));
		int cover = Integer.parseInt(line.get(54));

		// now build a training input
		input[0] = percent(elevation,analyze.getMaxElevation(),analyze.getMinElevation());
		input[1] = percent(aspect,analyze.getMaxAspect(),analyze.getMinAspect());
		input[2] = percent(slope,analyze.getMaxSlope(),analyze.getMinSlope());					
		input[3] = percent(hWater,analyze.getMaxHWater(),analyze.getMinHWater());
		input[4] = percent(vWater,analyze.getMaxVWater(),analyze.getMinVWater());
		input[5] = percent(road,analyze.getMaxRoad(),analyze.getMinRoad());
		input[6] = percent(shade9,analyze.getMaxShade9(),analyze.getMinShade9());
		input[7] = percent(shade12,analyze.getMaxShade12(),analyze.getMinShade12());
		input[8] = percent(shade3,analyze.getMaxShade3(),analyze.getMinShade3());

		if( ideal!=null )
		{
			for (int i = 0; i < Analyze.COVER_TYPES.length; i++)
				ideal[i] = Config.FALSE;
	
			ideal[cover - 1] = Config.TRUE;
		}
	}
}

Optimize.java Perform an incremental prune. Try to figure out an optimal number of hidden layers and neurons.

package com.heatonresearch.course.introneuralnet.program2;

import com.heatonresearch.book.introneuralnet.neural.feedforward.FeedforwardLayer;
import com.heatonresearch.book.introneuralnet.neural.feedforward.FeedforwardNetwork;

public class Optimize {
	private Analyze analyze; 
	private Train train;
	
	public FeedforwardNetwork create(int hidden1,int hidden2)
	{
		int input = this.train.getTraining()[0].length;
		int output = this.train.getIdeal()[0].length;
		
		final FeedforwardNetwork network = new FeedforwardNetwork();
		network.addLayer(new FeedforwardLayer(input));
		if( hidden1>0 )
			network.addLayer(new FeedforwardLayer(hidden1));
		if( hidden2>0 )
			network.addLayer(new FeedforwardLayer(hidden2));
		network.addLayer(new FeedforwardLayer(output));
		return network;
	}
	
	public FeedforwardNetwork optimize(Analyze analyze, Train train)
	{
		FeedforwardNetwork bestNetwork = null;
		double bestError = Double.MAX_VALUE;
		this.analyze = analyze;
		this.train = train;
		
		System.out.println("Searching for best hidden layer setup.");
		for(int hidden1=1;hidden1<Config.MAX_HIDDEN_LAYER_1;hidden1++)
		{
			for(int hidden2=0;hidden2<Config.MAX_HIDDEN_LAYER_2;hidden2++)
			{
				System.out.println("Trying: hidden1=" + hidden1 + ",hidden2=" + hidden2);
				FeedforwardNetwork trial = create(hidden1,hidden2);
				double error = train.train(trial);
				if( error<bestError)
				{
					bestNetwork = trial;
					bestError = error;
					System.out.println("New best network found: hidden1=" + hidden1
						+ ", hidden2="+ hidden2 + ", error=" + bestError);
				}
			}
		}
		
		return bestNetwork;
	}
}

Program2.java Main entry point for the program.

package com.heatonresearch.course.introneuralnet.program2;

import com.heatonresearch.book.introneuralnet.common.ReadCSV;
import com.heatonresearch.book.introneuralnet.neural.feedforward.FeedforwardLayer;
import com.heatonresearch.book.introneuralnet.neural.feedforward.FeedforwardNetwork;

public class Program2 {
	public static void main(String args[])
	{
		try
		{			
			Analyze analyze = new Analyze();
			analyze.analyze(Config.FILENAME);
			Train train = new Train();
			train.build(Config.FILENAME, analyze);
			Optimize optimize = new Optimize();
			FeedforwardNetwork network = optimize.optimize(analyze, train);
			if( network.getLayers().size()==3 )
			{
				FeedforwardLayer hidden1 = network.getLayers().get(1);
				System.out.println("The best network: hidden1 = " + hidden1.getNeuronCount());
			}
			else
			{
				FeedforwardLayer hidden1 = network.getLayers().get(1);
				FeedforwardLayer hidden2 = network.getLayers().get(2);
				System.out.println("The best network: hidden1 = " + hidden1.getNeuronCount()
						+",hidden2=" + hidden2.getNeuronCount());
			}
		}
		catch(Throwable t)
		{
			t.printStackTrace();
		}
	}
}

Train.java Train the neural network for each pruning cycle.

package com.heatonresearch.course.introneuralnet.program2;

import java.io.IOException;

import com.heatonresearch.book.introneuralnet.common.ReadCSV;
import com.heatonresearch.book.introneuralnet.neural.activation.ActivationTANH;
import com.heatonresearch.book.introneuralnet.neural.feedforward.FeedforwardLayer;
import com.heatonresearch.book.introneuralnet.neural.feedforward.FeedforwardNetwork;
import com.heatonresearch.book.introneuralnet.neural.feedforward.train.backpropagation.Backpropagation;

public class Train {

	private double[][] training;
	private double[][] ideal;
	private int [] coverCount;

	double percent(double value, double max, double min) {
		return (value - min) / (max - min);
	}

	public void build(String filename, Analyze analyze) throws IOException {

		this.training = new double[Config.TRAINING_SIZE][9];
		this.ideal = new double[Config.TRAINING_SIZE][Analyze.COVER_TYPES.length];
		this.coverCount = new int[Analyze.COVER_TYPES.length];
				
		int skip = (analyze.getCount() / Config.TRAINING_SIZE) - 1;
		int trainingIndex = 0;
		ReadCSV csv = new ReadCSV(filename, false);
		System.out.println("Generating training set, skip = " + skip);
		
		while (csv.next() && (trainingIndex < Config.TRAINING_SIZE)) {

			int cover = Integer.parseInt(csv.get(54))-1;
			
			if( coverCount[cover]<Config.COVER_SAMPLES )
			{
				Convert.convert(analyze, csv, training[trainingIndex],
					ideal[trainingIndex]);
				coverCount[cover]++;
				trainingIndex++;
			}			
		}
	}

	public double train(FeedforwardNetwork network) {
		
		// train the neural network
		final Backpropagation train = new Backpropagation(network,
				this.training, this.ideal, Config.LEARNING_RATE,
				Config.LEARNING_MOMENTUM);

		int epoch = 1;

		do {
			train.iteration();
			//System.out
			//		.println("Epoch #" + epoch + " Error:" + train.getError());
			epoch++;
		} while (epoch < Config.EPOCHS);
		return train.getError();
	}

	public double[][] getTraining() {
		return training;
	}

	public double[][] getIdeal() {
		return ideal;
	}
	
	
}

Videos for this Session

Video	Title
	Introduction to Neural Networks for Java(Class 14)