NanoNeuron

Definition

Definition

NanoNeuron is a simplified neural network algorithm designed to predict numerical values. It consists of a single neuron with one input and one output, incorporating basic principles of linear regression. The neuron has two parameters: weight (w) and bias (b). The algorithm aims to learn these parameters to fit a given dataset using gradient descent optimization

Explanation

Iteratively adjusting its weight and bias parameters to minimize the difference between predicted and actual values. Initially, random values are assigned to weight and bias. Then, for each data point, the predicted output is calculated by multiplying the input with the weight, adding the bias, and passing the result through an activation function (usually the identity function in this case). The algorithm calculates the loss (difference between predicted and actual output) and adjusts the parameters using gradient descent to minimize this loss. This process continues for a fixed number of iterations or until the loss converges to a satisfactory level

Guidance

• Initialize weight (w) and bias (b) with random values
• For each data point (input, output pair) in the dataset
  • calculate the predicted output (y_pred) using the formula: y_pred = w * input + b
  • compute the loss (error) between the predicted output and actual output
  • update the weight and bias using gradient descent
    • calculate the gradients of the loss function with respect to weight and bias
    • adjust weight and bias in the opposite direction of the gradients to minimize loss
    • update weight: w = w - learning_rate * dw
    • update bias: b = b - learning_rate * db
  • repeat steps for a fixed number of iterations or until convergence criteria are met
• Once training is complete, the neuron's weight and bias parameters represent the learned model

Tips

• choose an appropriate learning rate to balance convergence speed and stability
• monitor the loss during training to ensure it is decreasing steadily
• normalize the input data to a similar scale to aid convergence
• experiment with different activation functions and initialization methods for the neuron's parameters
• regularize the model if overfitting occurs by adding regularization terms to the loss function

Practice

Practice

# Initialize parameters
w = random()
b = random()
learning_rate = 0.01
num_iterations = 1000

# Training loop
for _ in range(num_iterations):
  # Iterate over dataset
  for input, output in dataset:
    # Forward pass
    y_pred = w * input + b
    # Compute loss
    loss = (y_pred - output)**2 / 2
    # Backward pass (gradient descent)
    dw = (y_pred - output) * input
    db = y_pred - output
    # Update parameters
    w = w - learning_rate * dw
    b = b - learning_rate * db

Solution

package main

import (
	"math"
)

type NanoNeuron struct {
	w float64
	b float64
}

func NewNanoNeuron(w, b float64) *NanoNeuron {
	return &NanoNeuron{w: w, b: b}
}

func (nn *NanoNeuron) Predict(x float64) float64 {
	return x*nn.w + nn.b
}

func CelsiusToFahrenheit(c float64) float64 {
	w := 1.8
	b := 32.0
	f := c*w + b
	return f
}

func GenerateDataSets() ([]float64, []float64, []float64, []float64) {
	xTrain := make([]float64, 0)
	yTrain := make([]float64, 0)

	for x := 0.0; x < 100; x += 1 {
		y := CelsiusToFahrenheit(x)
		xTrain = append(xTrain, x)
		yTrain = append(yTrain, y)
	}

	xTest := make([]float64, 0)
	yTest := make([]float64, 0)

	for x := 0.5; x < 100; x += 1 {
		y := CelsiusToFahrenheit(x)
		xTest = append(xTest, x)
		yTest = append(yTest, y)
	}

	return xTrain, yTrain, xTest, yTest
}

func PredictionCost(y, prediction float64) float64 {
	return math.Pow(y-prediction, 2) / 2
}

func ForwardPropagation(model *NanoNeuron, xTrain, yTrain []float64) ([]float64, float64) {
	m := len(xTrain)
	predictions := make([]float64, 0)
	cost := 0.0

	for i := 0; i < m; i++ {
		prediction := model.Predict(xTrain[i])
		cost += PredictionCost(yTrain[i], prediction)
		predictions = append(predictions, prediction)
	}

	cost /= float64(m)
	return predictions, cost
}

func BackwardPropagation(predictions, xTrain, yTrain []float64) (float64, float64) {
	m := len(xTrain)
	dW := 0.0
	dB := 0.0

	for i := 0; i < m; i++ {
		dW += (yTrain[i] - predictions[i]) * xTrain[i]
		dB += yTrain[i] - predictions[i]
	}

	dW /= float64(m)
	dB /= float64(m)

	return dW, dB
}

func TrainModel(model *NanoNeuron, epochs int, alpha float64, xTrain, yTrain []float64) []float64 {
	costHistory := make([]float64, 0)

	for epoch := 0; epoch < epochs; epoch++ {
		predictions, cost := ForwardPropagation(model, xTrain, yTrain)
		costHistory = append(costHistory, cost)

		dW, dB := BackwardPropagation(predictions, xTrain, yTrain)

		model.w += alpha * dW
		model.b += alpha * dB
	}

	return costHistory
}

import java.util.ArrayList;
import java.util.List;

class NanoNeuron {

  double w;
  double b;

  NanoNeuron(double w, double b) {
    this.w = w;
    this.b = b;
  }

  double predict(double x) {
    return x * this.w + this.b;
  }
}

public class Main {

  public static double celsiusToFahrenheit(double c) {
    final double w = 1.8;
    final double b = 32;
    return c * w + b;
  }

  public static List<List<Double>> generateDataSets() {
    List<Double> xTrain = new ArrayList<>();
    List<Double> yTrain = new ArrayList<>();
    for (double x = 0; x < 100; x += 1) {
      double y = celsiusToFahrenheit(x);
      xTrain.add(x);
      yTrain.add(y);
    }

    List<Double> xTest = new ArrayList<>();
    List<Double> yTest = new ArrayList<>();
    for (double x = 0.5; x < 100; x += 1) {
      double y = celsiusToFahrenheit(x);
      xTest.add(x);
      yTest.add(y);
    }

    List<List<Double>> dataSets = new ArrayList<>();
    dataSets.add(xTrain);
    dataSets.add(yTrain);
    dataSets.add(xTest);
    dataSets.add(yTest);
    return dataSets;
  }

  public static double predictionCost(double y, double prediction) {
    return Math.pow(y - prediction, 2) / 2;
  }

  public static double[] forwardPropagation(NanoNeuron model, List<Double> xTrain, List<Double> yTrain) {
    int m = xTrain.size();
    List<Double> predictions = new ArrayList<>();
    double cost = 0;
    for (int i = 0; i < m; i++) {
      double prediction = model.predict(xTrain.get(i));
      cost += predictionCost(yTrain.get(i), prediction);
      predictions.add(prediction);
    }
    cost /= m;
    return new double[]{cost, predictions};
  }

  public static double[] backwardPropagation(List<Double> predictions, List<Double> xTrain, List<Double> yTrain) {
    int m = xTrain.size();
    double dW = 0;
    double dB = 0;
    for (int i = 0; i < m; i++) {
      dW += (yTrain.get(i) - predictions.get(i)) * xTrain.get(i);
      dB += yTrain.get(i) - predictions.get(i);
    }
    dW /= m;
    dB /= m;
    return new double[]{dW, dB};
  }

  public static List<Double> trainModel(NanoNeuron model, int epochs, double alpha, List<Double> xTrain, List<Double> yTrain) {
    List<Double> costHistory = new ArrayList<>();
    for (int epoch = 0; epoch < epochs; epoch++) {
      double[] forwardResult = forwardPropagation(model, xTrain, yTrain);
      double cost = forwardResult[0];
      List<Double> predictions = forwardResult[1];

      double[] backwardResult = backwardPropagation(predictions, xTrain, yTrain);
      double dW = backwardResult[0];
      double dB = backwardResult[1];

      model.w += alpha * dW;
      model.b += alpha * dB;

      costHistory.add(cost);
    }
    return costHistory;
  }
}

function NanoNeuron(w, b) {
  this.w = w;
  this.b = b;
  this.predict = (x) => {
    return x * this.w + this.b;
  };
}

function celsiusToFahrenheit(c) {
  const w = 1.8;
  const b = 32;
  const f = c * w + b;
  return f;
}

function generateDataSets() {
  const xTrain = [];
  const yTrain = [];
  for (let x = 0; x < 100; x += 1) {
    const y = celsiusToFahrenheit(x);
    xTrain.push(x);
    yTrain.push(y);
  }

  const xTest = [];
  const yTest = [];
  for (let x = 0.5; x < 100; x += 1) {
    const y = celsiusToFahrenheit(x);
    xTest.push(x);
    yTest.push(y);
  }

  return [xTrain, yTrain, xTest, yTest];
}

function predictionCost(y, prediction) {
  return (y - prediction) ** 2 / 2;
}

function forwardPropagation(model, xTrain, yTrain) {
  const m = xTrain.length;
  const predictions = [];
  let cost = 0;
  for (let i = 0; i < m; i += 1) {
    const prediction = model.predict(xTrain[i]);
    cost += predictionCost(yTrain[i], prediction);
    predictions.push(prediction);
  }
  cost /= m;
  return [predictions, cost];
}

function backwardPropagation(predictions, xTrain, yTrain) {
  const m = xTrain.length;
  let dW = 0;
  let dB = 0;
  for (let i = 0; i < m; i += 1) {
    dW += (yTrain[i] - predictions[i]) * xTrain[i];
    dB += yTrain[i] - predictions[i];
  }
  dW /= m;
  dB /= m;
  return [dW, dB];
}

function trainModel({ model, epochs, alpha, xTrain, yTrain }) {
  const costHistory = [];
  for (let epoch = 0; epoch < epochs; epoch += 1) {
    const [predictions, cost] = forwardPropagation(model, xTrain, yTrain);
    costHistory.push(cost);

    const [dW, dB] = backwardPropagation(predictions, xTrain, yTrain);

    model.w += alpha * dW;
    model.b += alpha * dB;
  }
  return costHistory;
}

class NanoNeuron(val w: Double, val b: Double) {
    fun predict(x: Double): Double {
        return x * w + b
    }
}

fun celsiusToFahrenheit(c: Double): Double {
    val w = 1.8
    val b = 32
    return c * w + b
}

fun generateDataSets(): List<List<Double>> {
    val xTrain = mutableListOf<Double>()
    val yTrain = mutableListOf<Double>()
    for (x in 0 until 100) {
        val y = celsiusToFahrenheit(x.toDouble())
        xTrain.add(x.toDouble())
        yTrain.add(y)
    }

    val xTest = mutableListOf<Double>()
    val yTest = mutableListOf<Double>()
    for (x in 0.5 until 100) {
        val y = celsiusToFahrenheit(x)
        xTest.add(x)
        yTest.add(y)
    }

    return listOf(xTrain, yTrain, xTest, yTest)
}

fun predictionCost(y: Double, prediction: Double): Double {
    return Math.pow(y - prediction, 2.0) / 2
}

fun forwardPropagation(model: NanoNeuron, xTrain: List<Double>, yTrain: List<Double>): Pair<List<Double>, Double> {
    val m = xTrain.size
    val predictions = mutableListOf<Double>()
    var cost = 0.0
    for (i in 0 until m) {
        val prediction = model.predict(xTrain[i])
        cost += predictionCost(yTrain[i], prediction)
        predictions.add(prediction)
    }
    cost /= m
    return Pair(predictions, cost)
}

fun backwardPropagation(predictions: List<Double>, xTrain: List<Double>, yTrain: List<Double>): Pair<Double, Double> {
    val m = xTrain.size
    var dW = 0.0
    var dB = 0.0
    for (i in 0 until m) {
        dW += (yTrain[i] - predictions[i]) * xTrain[i]
        dB += yTrain[i] - predictions[i]
    }
    dW /= m
    dB /= m
    return Pair(dW, dB)
}

fun trainModel(model: NanoNeuron, epochs: Int, alpha: Double, xTrain: List<Double>, yTrain: List<Double>): List<Double> {
    val costHistory = mutableListOf<Double>()
    for (epoch in 0 until epochs) {
        val (predictions, cost) = forwardPropagation(model, xTrain, yTrain)
        costHistory.add(cost)

        val (dW, dB) = backwardPropagation(predictions, xTrain, yTrain)

        model.w += alpha * dW
        model.b += alpha * dB
    }
    return costHistory
}

class NanoNeuron:
    def __init__(self, w, b):
        self.w = w
        self.b = b

    def predict(self, x):
        return x * self.w + self.b

def celsius_to_fahrenheit(c):
    w = 1.8
    b = 32
    return c * w + b

def generate_data_sets():
    x_train = []
    y_train = []
    for x in range(100):
        y = celsius_to_fahrenheit(x)
        x_train.append(x)
        y_train.append(y)

    x_test = []
    y_test = []
    for x in range(1, 100):
        y = celsius_to_fahrenheit(x)
        x_test.append(x)
        y_test.append(y)

    return x_train, y_train, x_test, y_test

def prediction_cost(y, prediction):
    return (y - prediction) ** 2 / 2

def forward_propagation(model, x_train, y_train):
    m = len(x_train)
    predictions = []
    cost = 0
    for i in range(m):
        prediction = model.predict(x_train[i])
        cost += prediction_cost(y_train[i], prediction)
        predictions.append(prediction)
    cost /= m
    return predictions, cost

def backward_propagation(predictions, x_train, y_train):
    m = len(x_train)
    dW = 0
    dB = 0
    for i in range(m):
        dW += (y_train[i] - predictions[i]) * x_train[i]
        dB += y_train[i] - predictions[i]
    dW /= m
    dB /= m
    return dW, dB

def train_model(model, epochs, alpha, x_train, y_train):
    cost_history = []
    for epoch in range(epochs):
        predictions, cost = forward_propagation(model, x_train, y_train)
        cost_history.append(cost)

        dW, dB = backward_propagation(predictions, x_train, y_train)

        model.w += alpha * dW
        model.b += alpha * dB
    return cost_history

struct NanoNeuron {
    w: f64,
    b: f64,
}

impl NanoNeuron {
    fn new(w: f64, b: f64) -> Self {
        NanoNeuron { w, b }
    }

    fn predict(&self, x: f64) -> f64 {
        x * self.w + self.b
    }
}

fn celsius_to_fahrenheit(c: f64) -> f64 {
    let w = 1.8;
    let b = 32.0;
    c * w + b
}

fn generate_data_sets() -> (Vec<f64>, Vec<f64>, Vec<f64>, Vec<f64>) {
    let mut x_train = Vec::new();
    let mut y_train = Vec::new();
    for x in 0..100 {
        let y = celsius_to_fahrenheit(x as f64);
        x_train.push(x as f64);
        y_train.push(y);
    }

    let mut x_test = Vec::new();
    let mut y_test = Vec::new();
    for x in (0..100).map(|x| x as f64 + 0.5) {
        let y = celsius_to_fahrenheit(x);
        x_test.push(x);
        y_test.push(y);
    }

    (x_train, y_train, x_test, y_test)
}

fn prediction_cost(y: f64, prediction: f64) -> f64 {
    (y - prediction).powi(2) / 2.0
}

fn forward_propagation(model: &NanoNeuron, x_train: &[f64], y_train: &[f64]) -> (Vec<f64>, f64) {
    let m = x_train.len();
    let mut predictions = Vec::new();
    let mut cost = 0.0;
    for i in 0..m {
        let prediction = model.predict(x_train[i]);
        cost += prediction_cost(y_train[i], prediction);
        predictions.push(prediction);
    }
    cost /= m as f64;
    (predictions, cost)
}

fn backward_propagation(predictions: &[f64], x_train: &[f64], y_train: &[f64]) -> (f64, f64) {
    let m = x_train.len() as f64;
    let (mut dW, mut dB) = (0.0, 0.0);
    for i in 0..x_train.len() {
        dW += (y_train[i] - predictions[i]) * x_train[i];
        dB += y_train[i] - predictions[i];
    }
    dW /= m;
    dB /= m;
    (dW, dB)
}

fn train_model(model: &mut NanoNeuron, epochs: usize, alpha: f64, x_train: &[f64], y_train: &[f64]) -> Vec<f64> {
    let mut cost_history = Vec::new();
    for _ in 0..epochs {
        let (predictions, cost) = forward_propagation(model, x_train, y_train);
        cost_history.push(cost);

        let (dW, dB) = backward_propagation(&predictions, x_train, y_train);

        model.w += alpha * dW;
        model.b += alpha * dB;
    }
    cost_history
}

class NanoNeuron {
  w: number;
  b: number;

  constructor(w: number, b: number) {
    this.w = w;
    this.b = b;
  }

  predict(x: number): number {
    return x * this.w + this.b;
  }
}

function celsiusToFahrenheit(c: number): number {
  const w: number = 1.8;
  const b: number = 32;
  const f: number = c * w + b;
  return f;
}

function generateDataSets(): [number[], number[], number[], number[]] {
  const xTrain: number[] = [];
  const yTrain: number[] = [];
  for (let x = 0; x < 100; x += 1) {
    const y: number = celsiusToFahrenheit(x);
    xTrain.push(x);
    yTrain.push(y);
  }

  const xTest: number[] = [];
  const yTest: number[] = [];
  for (let x = 0.5; x < 100; x += 1) {
    const y: number = celsiusToFahrenheit(x);
    xTest.push(x);
    yTest.push(y);
  }

  return [xTrain, yTrain, xTest, yTest];
}

function predictionCost(y: number, prediction: number): number {
  return Math.pow(y - prediction, 2) / 2;
}

function forwardPropagation(
  model: NanoNeuron,
  xTrain: number[],
  yTrain: number[],
): [number[], number] {
  const m: number = xTrain.length;
  const predictions: number[] = [];
  let cost: number = 0;
  for (let i = 0; i < m; i += 1) {
    const prediction: number = model.predict(xTrain[i]);
    cost += predictionCost(yTrain[i], prediction);
    predictions.push(prediction);
  }
  cost /= m;
  return [predictions, cost];
}

function backwardPropagation(
  predictions: number[],
  xTrain: number[],
  yTrain: number[],
): [number, number] {
  const m: number = xTrain.length;
  let dW: number = 0;
  let dB: number = 0;
  for (let i = 0; i < m; i += 1) {
    dW += (yTrain[i] - predictions[i]) * xTrain[i];
    dB += yTrain[i] - predictions[i];
  }
  dW /= m;
  dB /= m;
  return [dW, dB];
}

function trainModel({
  model,
  epochs,
  alpha,
  xTrain,
  yTrain,
}: {
  model: NanoNeuron;
  epochs: number;
  alpha: number;
  xTrain: number[];
  yTrain: number[];
}): number[] {
  const costHistory: number[] = [];
  for (let epoch = 0; epoch < epochs; epoch += 1) {
    const [predictions, cost]: [number[], number] = forwardPropagation(
      model,
      xTrain,
      yTrain,
    );
    costHistory.push(cost);

    const [dW, dB]: [number, number] = backwardPropagation(
      predictions,
      xTrain,
      yTrain,
    );

    model.w += alpha * dW;
    model.b += alpha * dB;
  }
  return costHistory;
}