diff --git a/Backend/Machine Learning/Networks/network.cpp b/Backend/Machine Learning/Networks/network.cpp index cf0e6aaee..68f6f80d3 100644 --- a/Backend/Machine Learning/Networks/network.cpp +++ b/Backend/Machine Learning/Networks/network.cpp @@ -41,7 +41,7 @@ using namespace glades; * @brief NNetwork constructor * @details creates an empty nnetwork */ -glades::NNetwork::NNetwork(int newNetType) +glades::NNetwork::NNetwork() { running = false; di = NULL; @@ -49,7 +49,7 @@ glades::NNetwork::NNetwork(int newNetType) serverInstance = NULL; cConnection = NULL; clean(); - netType = newNetType; + netType = TYPE_DFF; minibatchSize = NNInfo::BATCH_STOCHASTIC; } @@ -57,7 +57,7 @@ glades::NNetwork::NNetwork(int newNetType) * @brief NNetwork destructor * @details destroys the NNetwork object */ -glades::NNetwork::NNetwork(NNInfo* newNNInfo, int newNetType) +glades::NNetwork::NNetwork(NNInfo* newNNInfo) { if (!newNNInfo) return; @@ -69,7 +69,7 @@ glades::NNetwork::NNetwork(NNInfo* newNNInfo, int newNetType) cConnection = NULL; clean(); skeleton = newNNInfo; - netType = newNetType; + netType = TYPE_DFF; minibatchSize = skeleton->getBatchSize(); } @@ -101,16 +101,235 @@ void glades::NNetwork::stop() running = false; } -void glades::NNetwork::train(DataInput* newDataInput) -{ - run(newDataInput, RUN_TRAIN); +void glades::NNetwork::train(DataInput* newDataInput) { + if (!skeleton) return; + if (!newDataInput) return; + + di = newDataInput; + if ((di->getTrainSize() <= 0) || (di->getFeatureCount() <= 0)) return; + + // Build the network if it hasn't been built yet + if (epochs == 0) + meat.build(skeleton, di, netType); + + // Set the mini batch size + minibatchSize = skeleton->getBatchSize(); + + // Check if layers are valid + if ((meat.getInputLayersSize() <= 0) || (meat.getLayersSize() <= 0)) return; + + // Build empty confusion matrix if needed + if ((skeleton->getOutputType() == GMath::CLASSIFICATION) || + (skeleton->getOutputType() == GMath::KL)) + confusionMatrix.build(skeleton->getOutputLayerSize()); + + // Reset the different graphs e.g. learning curve + resetGraphs(); + + // Training loop + running = true; + firstRunActivation = false; + int64_t lastUpdateTime = 0; + + while (running) { + // Global network statistics + overallTotalError = 0.0f; + overallTotalAccuracy = 0.0f; + + // Reset confusion matrix if needed + if ((skeleton->getOutputType() == GMath::CLASSIFICATION) || + (skeleton->getOutputType() == GMath::KL)) + confusionMatrix.reset(); + + // Recursive Forward Pass and Back Propagation + for (unsigned int r = 0; r < meat.getInputLayersSize(); ++r) + SGDHelper(r, RUN_TRAIN); + + // Update the network vars + ++epochs; + overallTotalAccuracy /= ((float)meat.getInputLayersSize()) * ((float)skeleton->getOutputLayerSize()); + + // Display and debugging + if (skeleton->getOutputType() == GMath::REGRESSION) { + printf("\33[2K[NN] %d\t%f%%\r", epochs, overallTotalAccuracy); + fflush(stdout); + } else if ((skeleton->getOutputType() == GMath::CLASSIFICATION) || + (skeleton->getOutputType() == GMath::KL)) { + confusionMatrix.updateResultParams(); + + overallClassAccuracy = (confusionMatrix.getOverallAccuracy() * 100.0f); + overallClassPrecision = (confusionMatrix.getOverallPrecision() * 100.0f); + overallClassRecall = (confusionMatrix.getOverallRecall() * 100.0f); + overallClassSpecificity = (confusionMatrix.getOverallSpecificity() * 100.0f); + overallClassF1 = confusionMatrix.getOverallF1Score() * 100.0f; + + if (epochs < 100) { + printf("\33[2K[NN] %d\t\t%f%%\t%f%%\t%f%%\t%f%%\t%f%%\r", epochs, + overallClassAccuracy, overallClassPrecision, overallClassRecall, + overallClassSpecificity, overallClassF1); + fflush(stdout); + } else { + printf("\33[2K[NN] %d\t%f%%\t%f%%\t%f%%\t%f%%\t%f%%\r", epochs, + overallClassAccuracy, overallClassPrecision, overallClassRecall, + overallClassSpecificity, overallClassF1); + fflush(stdout); + } + } + + // Update the GUI with the metrics + int64_t ms = getCurrentTimeMilliseconds(); + int64_t timeDiff = ms - lastUpdateTime; + if ((serverInstance && cConnection) && ((epochs < 10) || (timeDiff > 16))) { // 60fps + if (epochs > 0) { + // Update the learning curve + shmea::GList wData; + wData.addInt(epochs - 1); // -1 because we don't plot the first point + wData.addFloat(overallTotalError); + + shmea::GList argData; + argData.addString("PROGRESSIVE"); + + shmea::ServiceData* cData = new shmea::ServiceData(cConnection, "GUI_Callback"); + cData->set(wData); + cData->setArgList(argData); + serverInstance->send(cData); + + // Update the weights of the Neural Network + argData.clear(); + argData.addString("ACTIVATIONS"); + cData = new shmea::ServiceData(cConnection, "GUI_Callback"); + if (!firstRunActivation) { + firstRunActivation = true; + shmea::GList layerSizes; + for (unsigned int cLayerCounter = 0; cLayerCounter < skeleton->numHiddenLayers() + 2; ++cLayerCounter) { + layerSizes.addInt(meat.getLayerSize(cLayerCounter)); + } + cData->set(layerSizes); + cData->setArgList(argData); + } else { + cData->set(cNodeActivations); + cData->setArgList(argData); + } + serverInstance->send(cData); + + argData.clear(); + shmea::GList obtainedWeights = meat.getWeights(); + argData.addString("WEIGHTS"); + cData = new shmea::ServiceData(cConnection, "GUI_Callback"); + cData->set(obtainedWeights); + cData->setArgList(argData); + serverInstance->send(cData); + } + + // Update the accuracy label + { + shmea::GList argData; + argData.addString("ACC"); + + shmea::GList wData; + wData.addInt(epochs); + wData.addFloat(overallTotalAccuracy); + + shmea::ServiceData* cData = new shmea::ServiceData(cConnection, "GUI_Callback"); + cData->set(wData); + cData->setArgList(argData); + serverInstance->send(cData); + } + + if ((skeleton->getOutputType() == GMath::CLASSIFICATION) || + (skeleton->getOutputType() == GMath::KL)) { + // Update the ROC Curve and Conf Matrix + shmea::GList argData; + argData.addString("CONF"); + argData.addFloat(confusionMatrix.getOverallFalseAlarm()); + argData.addFloat(confusionMatrix.getOverallRecall()); + + shmea::ServiceData* cData = new shmea::ServiceData(cConnection, "GUI_Callback"); + cData->set(confusionMatrix.getMatrix()); + cData->setArgList(argData); + serverInstance->send(cData); + } + + lastUpdateTime = ms; + } + + cNodeActivations.clear(); + + // Shut it down? + if (terminator.triggered(time(NULL), epochs, overallTotalAccuracy)) + break; + } + + // For the carriage controlled print + printf("\n"); + + // Print the results + meat.print(skeleton); + + // Clean confusion matrix + if ((skeleton->getOutputType() == GMath::CLASSIFICATION) || + (skeleton->getOutputType() == GMath::KL)) + confusionMatrix.clean(); + + printf("\n"); + + // So the network doesn't immediately quit next time and we can prematurely start our net + running = false; } -void glades::NNetwork::test(DataInput* newDataInput) -{ - run(newDataInput, RUN_TEST); + +void glades::NNetwork::test(DataInput* newDataInput) { + if (!skeleton) return; + if (!newDataInput) return; + + di = newDataInput; + if ((di->getTrainSize() <= 0) || (di->getFeatureCount() <= 0)) return; + + // Build the network if it hasn't been built yet + if (epochs == 0) + meat.build(skeleton, di, netType); + + // Set the mini batch size + minibatchSize = skeleton->getBatchSize(); + + // Check if layers are valid + if ((meat.getInputLayersSize() <= 0) || (meat.getLayersSize() <= 0)) return; + + // Build empty confusion matrix if needed + if ((skeleton->getOutputType() == GMath::CLASSIFICATION) || + (skeleton->getOutputType() == GMath::KL)) + confusionMatrix.build(skeleton->getOutputLayerSize()); + + // Reset the different graphs e.g. learning curve + resetGraphs(); + + // Testing loop + running = true; + firstRunActivation = false; + int64_t lastUpdateTime = 0; + + // Run the test + for (unsigned int r = 0; r < meat.getInputLayersSize(); ++r) + SGDHelper(r, RUN_TEST); + + // Update the network vars and print + printf("[NN] %s Accuracy: %f%%\n", skeleton->getName().c_str(), overallTotalAccuracy); + + // Print the results + meat.print(skeleton); + + // Clean confusion matrix + if ((skeleton->getOutputType() == GMath::CLASSIFICATION) || + (skeleton->getOutputType() == GMath::KL)) + confusionMatrix.clean(); + + printf("\n"); + + // So the network doesn't immediately quit next time and we can prematurely start our net + running = false; } + void glades::NNetwork::run(DataInput* newDataInput, int runType) { if (!skeleton) @@ -145,7 +364,7 @@ void glades::NNetwork::run(DataInput* newDataInput, int runType) confusionMatrix.build(skeleton->getOutputLayerSize()); // if (DEBUG_ADVANCED) - //meat.print(skeleton, true); + // meat.print(skeleton); if (runType == RUN_TRAIN) printf("[NN] Training...\n"); @@ -160,7 +379,7 @@ void glades::NNetwork::run(DataInput* newDataInput, int runType) if ((skeleton->getOutputType() == GMath::CLASSIFICATION) || (skeleton->getOutputType() == GMath::KL)) - printf("[NN] Epochs\tAccuracy\tMCC\t\tPrecision\tRecall\t\tSpecificity\tF1 Score\n"); + printf("[NN] Epochs\tAccuracy\tPrecision\tRecall\t\tSpecificity\tF1 Score\n"); } // Reset the different graphcs e.g. learning curve @@ -222,7 +441,6 @@ void glades::NNetwork::run(DataInput* newDataInput, int runType) overallClassRecall = (confusionMatrix.getOverallRecall() * 100.0f); overallClassSpecificity = (confusionMatrix.getOverallSpecificity() * 100.0f); overallClassF1 = confusionMatrix.getOverallF1Score() * 100.0f; - float mcc = confusionMatrix.getOverallMCC(); // Display and debugging if (runType == RUN_TRAIN) @@ -231,14 +449,14 @@ void glades::NNetwork::run(DataInput* newDataInput, int runType) { if (epochs < 100) { - printf("\33[2K[NN] %d\t\t%f%%\t%f%%\t%f%%\t%f%%\t%f%%\t%f%%\r", epochs, - overallClassAccuracy, mcc, overallClassPrecision, overallClassRecall, + printf("\33[2K[NN] %d\t\t%f%%\t%f%%\t%f%%\t%f%%\t%f%%\r", epochs, + overallClassAccuracy, overallClassPrecision, overallClassRecall, overallClassSpecificity, overallClassF1); fflush(stdout); } else { - printf("\33[2K[NN] %d\t%f%%\t%f%%\t%f%%\t%f%%\t%f%%\t%f%%\r", epochs, + printf("\33[2K[NN] %d\t%f%%\t%f%%\t%f%%\t%f%%\t%f%%\r", epochs, overallClassAccuracy, overallClassPrecision, overallClassRecall, overallClassSpecificity, overallClassF1); fflush(stdout); @@ -455,243 +673,157 @@ void glades::NNetwork::SGDHelper(unsigned int inputRowCounter, int runType) //printf("=========================================\n"); } -void glades::NNetwork::ForwardPass(unsigned int inputRowCounter, - int cInputLayerCounter, int cOutputLayerCounter, - unsigned int cInputNodeCounter, unsigned int cOutputNodeCounter) -{ - for(unsigned int cLayerCounter = 0; cLayerCounter < skeleton->numHiddenLayers()+1; ++cLayerCounter) - { - cInputLayerCounter = cLayerCounter; - cOutputLayerCounter = cLayerCounter + 1; - - for(cOutputNodeCounter = 0; cOutputNodeCounter < meat.getLayerSize(cOutputLayerCounter); ++cOutputNodeCounter) - { - for(cInputNodeCounter = 0; cInputNodeCounter < meat.getLayerSize(cInputLayerCounter); ++cInputNodeCounter) - { - NetworkState* netState = - meat.getNetworkStateFromLoc(inputRowCounter, cInputLayerCounter, cOutputLayerCounter, - cInputNodeCounter, cOutputNodeCounter); - if (!netState) - return; - - //printf("ForwardPass: %d %d %d %d %d\n", inputRowCounter, cInputLayerCounter, cOutputLayerCounter, - //cInputNodeCounter, cOutputNodeCounter); - - // Does Dropout occur? - bool dropout = (!((netState->validInputNode) && (netState->validOutputNode))); - - // Add the input node activation to the output node - if (!dropout) - { - float cEdgeActivation = netState->cOutputNode->getEdgeWeight(cInputNodeCounter) * - netState->cInputNode->getWeight(); - //printf("\n\nEdge Activation[%d][%d][%d][%d][%d]: %f\n\n", inputRowCounter, cInputLayerCounter, cOutputLayerCounter, cInputNodeCounter, cOutputNodeCounter, cEdgeActivation); - netState->cOutputNode->setActivation(cInputNodeCounter, cEdgeActivation); - } - // Last Input Node for the Output Node - float cOutputLayerActivation = 0.0f; - if (netState->lastValidInputNode) - { - // Get the current output node activation - float cOutputNodeActivation = netState->cOutputNode->getActivation(); - - // Context Nodes - if ((netType == TYPE_RNN) && (netState->cOutputLayer->getType() == Layer::HIDDEN_TYPE)) - { - float cContextEdgeActivation = - netState->cOutputNode->getContextNode()->getEdgeWeight(0) * - netState->cOutputNode->getContextNode()->getWeight(); - cOutputNodeActivation += cContextEdgeActivation; - } - - // Add the bias if we are in a hidden layer or output layer - // Input Layer fundamentally cannot have a bias - if (netState->cInputLayer->getType() != Layer::INPUT_TYPE) - cOutputNodeActivation += netState->cInputLayer->getBiasWeight(); - - if ((netType == TYPE_RNN) && (netState->cOutputLayer->getType() == Layer::HIDDEN_TYPE)) - { - netState->cOutputNode->getContextNode()->setWeight(netState->cOutputNode->getActivation()); - } - - // Clean the output node activation for next run (cleanup) - netState->cOutputNode->clearActivation(); - - // Set Our prediction based on the cOutputNode activation - int cActivationFx = skeleton->getActivationType(cInputLayerCounter); - float cActivationParam = skeleton->getActivationParam(cInputLayerCounter); - cOutputLayerActivation = - GMath::squash(cOutputNodeActivation, cActivationFx, cActivationParam); - - //We add the current node activation to the list of activations that will be sent on the network for visualization purposes - if(inputRowCounter == di->getTrainSize()-1) - { - cNodeActivations.addFloat(cOutputNodeActivation); - } - - netState->cOutputNode->setWeight(cOutputLayerActivation); - - // Output layer calculations - if (netState->cOutputLayer->getType() == Layer::OUTPUT_TYPE) - { - // Get the prediction and expected vars - float prediction = netState->cOutputNode->getWeight(); - float expectation = - di->getTrainExpectedRow(inputRowCounter)[cOutputNodeCounter]; - //printf("Expectation: %f Prediction: %f\n", expectation, prediction); - - // Add the expected and predicted to the result row - results.addFloat(expectation); - results.addFloat(prediction); - - // Cost function calculations - float dataSize = (float)(di->getTrainSize() * netState->cOutputLayer->size()); - int costFx = skeleton->getOutputType(); - float cOutputCost = GMath::outputNodeCost(expectation, prediction, dataSize, costFx); - - // Error across every input instance - overallTotalError += cOutputCost; - - // Accuracy vars - float percentError = GMath::PercentError(prediction, expectation, cOutputCost); - float calculatedError = GMath::error(expectation, prediction); - bool isCorrect = percentError < GMath::OUTLIER; - float accuracy = (1.0f - percentError) * 100.0f; - if (accuracy < 0.0f) - accuracy = 0.0f; - overallTotalAccuracy += accuracy; - - // Advanced Debugging - /*if (DEBUG_ADVANCED) - { - printf("%f\t%f\t%f\t%f\t%f%%\t(%s)\n", expectation, prediction, calculatedError, - cOutputCost, (1.0f - percentError) * 100.0f, isCorrect ? "True" : "False"); - - // Multiple output nodes - if (netState->cOutputLayer->size() > 1) - printf("-----------------------------------------------------------\n"); - }*/ - } - } - - - - - delete netState; - } - - - } - if(inputRowCounter == di->getTrainSize()-1) - { - cNodeActivations.addString(","); - } - } +void glades::NNetwork::ForwardPass(unsigned int inputRowCounter, int cInputLayerCounter, int cOutputLayerCounter, unsigned int cInputNodeCounter, unsigned int cOutputNodeCounter) { + // Modify the method to support 1D convolutional operations + // Ensure the method handles context windows during convolution + for (unsigned int cLayerCounter = 0; cLayerCounter < skeleton->numHiddenLayers() + 1; ++cLayerCounter) { + cInputLayerCounter = cLayerCounter; + cOutputLayerCounter = cLayerCounter + 1; + + for (cOutputNodeCounter = 0; cOutputNodeCounter < meat.getLayerSize(cOutputLayerCounter); ++cOutputNodeCounter) { + for (cInputNodeCounter = 0; cInputNodeCounter < meat.getLayerSize(cInputLayerCounter); ++cInputNodeCounter) { + NetworkState* netState = meat.getNetworkStateFromLoc(inputRowCounter, cInputLayerCounter, cOutputLayerCounter, cInputNodeCounter, cOutputNodeCounter); + if (!netState) return; + + // Perform 1D convolution + std::vector inputActivation = netState->cInputNode->getActivation(); + std::vector outputActivation; + netState->cOutputLayer->perform1DConvolution(inputActivation, outputActivation, contextWindowSize); + + // Add the input node activation to the output node + if (!dropout) { + float cEdgeActivation = netState->cOutputNode->getEdgeWeight(cInputNodeCounter) * netState->cInputNode->getWeight(); + netState->cOutputNode->setActivation(cInputNodeCounter, cEdgeActivation); + } + + // Last Input Node for the Output Node + float cOutputLayerActivation = 0.0f; + if (netState->lastValidInputNode) { + // Get the current output node activation + float cOutputNodeActivation = netState->cOutputNode->getActivation(); + + // Clean the output node activation for next run (cleanup) + netState->cOutputNode->clearActivation(); + + // Add the bias if we are in a hidden layer or output layer + if (netState->cInputLayer->getType() != Layer::INPUT_TYPE) + cOutputNodeActivation += netState->cInputLayer->getBiasWeight(); + + // Set Our prediction based on the cOutputNode activation + int cActivationFx = skeleton->getActivationType(cInputLayerCounter); + float cActivationParam = skeleton->getActivationParam(cInputLayerCounter); + cOutputLayerActivation = GMath::squash(cOutputNodeActivation, cActivationFx, cActivationParam); + + netState->cOutputNode->setWeight(cOutputLayerActivation); + + // Output layer calculations + if (netState->cOutputLayer->getType() == Layer::OUTPUT_TYPE) { + // Get the prediction and expected vars + float prediction = netState->cOutputNode->getWeight(); + shmea::GType expectedCell = di->getTrainExpectedRow(inputRowCounter)[cOutputNodeCounter]; + float expectation = expectedCell; + + // Add the expected and predicted to the result row + results.addFloat(expectation); + results.addFloat(prediction); + + // Cost function calculations + float dataSize = (float)(di->getTrainSize() * netState->cOutputLayer->size()); + int costFx = skeleton->getOutputType(); + float cOutputCost = GMath::outputNodeCost(expectation, prediction, dataSize, costFx); + + // Error across every input instance + overallTotalError += cOutputCost; + + // Accuracy vars + float percentError = GMath::PercentError(prediction, expectation, cOutputCost); + float calculatedError = GMath::error(expectation, prediction); + bool isCorrect = percentError < GMath::OUTLIER; + float accuracy = (1.0f - percentError) * 100.0f; + if (accuracy < 0.0f) accuracy = 0.0f; + overallTotalAccuracy += accuracy; + } + } + + delete netState; + } + } + } } -void glades::NNetwork::BackPropagation(unsigned int inputRowCounter, int cInputLayerCounter, - int cOutputLayerCounter, unsigned int cInputNodeCounter, - unsigned int cOutputNodeCounter) -{ - for(unsigned int cLayerCounter = skeleton->numHiddenLayers()+1; cLayerCounter > 0; --cLayerCounter) - { - cOutputLayerCounter = cLayerCounter; - cInputLayerCounter = cLayerCounter - 1; - for(cInputNodeCounter = 0; cInputNodeCounter < meat.getLayerSize(cInputLayerCounter); ++cInputNodeCounter) - { - for(cOutputNodeCounter = 0; cOutputNodeCounter < meat.getLayerSize(cOutputLayerCounter); ++cOutputNodeCounter) - { - NetworkState* netState = - meat.getNetworkStateFromLoc(inputRowCounter, cInputLayerCounter, cOutputLayerCounter, - cInputNodeCounter, cOutputNodeCounter); - if (!netState) - return; - - //printf("BackPropagation: %d %d %d %d %d\n", inputRowCounter, cInputLayerCounter, - //cOutputLayerCounter, cInputNodeCounter, cOutputNodeCounter); - - // Output Layer Error Derivative Calculation - float cOutputDer = 1.0f; // Output der is linear so its 1 - if (netState->cOutputLayer->getType() == Layer::OUTPUT_TYPE) - { - // Cost function error derivative for output layer(s) - float prediction = netState->cOutputNode->getWeight(); - float expectation = - di->getTrainExpectedRow(inputRowCounter)[cOutputNodeCounter]; - - int costFx = skeleton->getOutputType(); - netState->cOutputNode->clearErrDer(); - netState->cOutputNode->adjustErrDer(GMath::costErrDer(expectation, prediction, costFx)); - } - else if (netState->cOutputLayer->getType() == Layer::HIDDEN_TYPE) - { - // Activation error derivative - int cActivationFx = skeleton->getActivationType(cInputLayerCounter); - cOutputDer = - GMath::activationErrDer(netState->cOutputNode->getWeight(), cActivationFx, 0.01f); - } - - // Does Dropout occur? - bool dropout = (!((netState->validInputNode) && (netState->validOutputNode))); - - // Node error derivative - float cOutNetErrDer = netState->cOutputNode->getErrDer(); - if (!dropout) - { - cOutNetErrDer *= cOutputDer; // current error partial - - // Clean the output node err der (cleanup) - if (cInputNodeCounter == netState->cInputLayer->size() - 1) - netState->cOutputNode->clearErrDer(); - - // MSE applied through gradient descent - float learningRate = skeleton->getLearningRate(cInputLayerCounter); - float momentumFactor = skeleton->getMomentumFactor(cInputLayerCounter); - float weightDecay1 = skeleton->getWeightDecay1(cInputLayerCounter); - float weightDecay2 = skeleton->getWeightDecay2(cInputLayerCounter); - float baseError = learningRate * cOutNetErrDer; - - // Add the weight delta - netState->cOutputNode->getDelta(cInputNodeCounter, baseError, - netState->cInputNode->getWeight(), learningRate, - momentumFactor, weightDecay1, weightDecay2); - - if((netType == TYPE_RNN) && (netState->cOutputLayer->getType() == Layer::HIDDEN_TYPE)) - { - netState->cOutputNode->getContextNode()->getDelta(0, baseError, - netState->cOutputNode->getContextNode()->getWeight(), learningRate, momentumFactor, weightDecay1, weightDecay2); - } - - // Apply all deltas if we've hit the minibatch size - if ((inputRowCounter % minibatchSize) == 0) - { - if((netType == TYPE_RNN) && (netState->cOutputLayer->getType() == Layer::HIDDEN_TYPE)) - { - netState->cOutputNode->getContextNode()->applyDeltas(0, minibatchSize); - netState->cOutputNode->getContextNode()->clearPrevDeltas(0); - } - - netState->cOutputNode->applyDeltas(cInputNodeCounter, minibatchSize); - netState->cOutputNode->clearPrevDeltas(cInputNodeCounter); - } - - // Update the bias (inputs fundamentally cannot have a bias) - if (netState->cInputLayer->getType() != Layer::INPUT_TYPE) - netState->cInputLayer->setBiasWeight(netState->cInputLayer->getBiasWeight() - - baseError); - } - - // Update the error partials for the next recursive calls - float cInNetErrDer = netState->cInputNode->getErrDer() + - (cOutNetErrDer * netState->cOutputNode->getEdgeWeight(cInputNodeCounter)); - netState->cInputNode->adjustErrDer(cInNetErrDer); - - delete netState; - } - } - } +void glades::NNetwork::BackPropagation(unsigned int inputRowCounter, int cInputLayerCounter, int cOutputLayerCounter, unsigned int cInputNodeCounter, unsigned int cOutputNodeCounter) { + // Modify the method to support 1D convolutional operations + // Ensure the method handles context windows during convolution + for (unsigned int cLayerCounter = skeleton->numHiddenLayers() + 1; cLayerCounter > 0; --cLayerCounter) { + cOutputLayerCounter = cLayerCounter; + cInputLayerCounter = cLayerCounter - 1; + for (cInputNodeCounter = 0; cInputNodeCounter < meat.getLayerSize(cInputLayerCounter); ++cInputNodeCounter) { + for (cOutputNodeCounter = 0; cOutputNodeCounter < meat.getLayerSize(cOutputLayerCounter); ++cOutputNodeCounter) { + NetworkState* netState = meat.getNetworkStateFromLoc(inputRowCounter, cInputLayerCounter, cOutputLayerCounter, cInputNodeCounter, cOutputNodeCounter); + if (!netState) return; + + // Output Layer Error Derivative Calculation + float cOutputDer = 1.0f; // Output der is linear so its 1 + if (netState->cOutputLayer->getType() == Layer::OUTPUT_TYPE) { + // Cost function error derivative for output layer(s) + float prediction = netState->cOutputNode->getWeight(); + float expectation = 0.0f; + shmea::GType expectedCell = di->getTrainExpectedRow(inputRowCounter)[cOutputNodeCounter]; + expectation = expectedCell.getFloat(); + + int costFx = skeleton->getOutputType(); + netState->cOutputNode->clearErrDer(); + netState->cOutputNode->adjustErrDer(GMath::costErrDer(expectation, prediction, costFx)); + } else if (netState->cOutputLayer->getType() == Layer::HIDDEN_TYPE) { + // Activation error derivative + int cActivationFx = skeleton->getActivationType(cInputLayerCounter); + cOutputDer = GMath::activationErrDer(netState->cOutputNode->getWeight(), cActivationFx, 0.01f); + } + + // Does Dropout occur? + bool dropout = (!((netState->validInputNode) && (netState->validOutputNode))); + + // Node error derivative + float cOutNetErrDer = netState->cOutputNode->getErrDer(); + if (!dropout) { + cOutNetErrDer *= cOutputDer; // current error partial + + // Clean the output node err der (cleanup) + if (cInputNodeCounter == netState->cInputLayer->size() - 1) + netState->cOutputNode->clearErrDer(); + + // MSE applied through gradient descent + float learningRate = skeleton->getLearningRate(cInputLayerCounter); + float momentumFactor = skeleton->getMomentumFactor(cInputLayerCounter); + float weightDecay1 = skeleton->getWeightDecay1(cInputLayerCounter); + float weightDecay2 = skeleton->getWeightDecay2(cInputLayerCounter); + float baseError = learningRate * cOutNetErrDer; + + // Add the weight delta + netState->cOutputNode->getDelta(cInputNodeCounter, baseError, netState->cInputNode->getWeight(), learningRate, momentumFactor, weightDecay1, weightDecay2); + + // Apply all deltas if we've hit the minibatch size + if ((inputRowCounter % minibatchSize) == 0) { + netState->cOutputNode->applyDeltas(cInputNodeCounter, minibatchSize); + netState->cOutputNode->clearPrevDeltas(cInputNodeCounter); + } + + // Update the bias (inputs fundamentally cannot have a bias) + if (netState->cInputLayer->getType() != Layer::INPUT_TYPE) + netState->cInputLayer->setBiasWeight(netState->cInputLayer->getBiasWeight() - baseError); + } + + // Update the error partials for the next recursive calls + float cInNetErrDer = netState->cInputNode->getErrDer() + (cOutNetErrDer * netState->cOutputNode->getEdgeWeight(cInputNodeCounter)); + netState->cInputNode->adjustErrDer(cInNetErrDer); + + delete netState; + } + } + } } + /*! * @brief get ID * @details get the network's unique ID @@ -752,6 +884,16 @@ void glades::NNetwork::setServer(GNet::GServer* newServer, GNet::Connection* new cConnection = newConnection; } +shmea::GList glades::NNetwork::getLearningCurve() const +{ + return learningCurve; +} + +// const std::vector& NNetwork::getROCCurve() const +// { +// return rocCurve; +// } + shmea::GList glades::NNetwork::getResults() const { return results; @@ -778,6 +920,32 @@ void glades::NNetwork::clean() void glades::NNetwork::resetGraphs() { + learningCurve.clear(); + + /*for (unsigned int i = 0; i < rocCurve.size(); ++i) + delete rocCurve[i];*/ + rocCurve.clear(); + rocCurve.reserve(10000); // arbitrary number + // create the results again results.clear(); } +void glades::NNetwork::setInputData(const std::vector>& images, const std::vector& labels) { + + meat.cleanInputLayers(); + + // Iterate over each image in the dataset + for (const auto& image : images) { + Layer* inputLayer = new Layer(Layer::INPUT_TYPE); + + // Iterate over each pixel in the image + for (float pixel : image) { + Node* node = new Node(); + node->setWeight(pixel); + inputLayer->addNode(node); + } + meat.addInputLayer(inputLayer); + } + + +} \ No newline at end of file diff --git a/Backend/Machine Learning/Networks/network.h b/Backend/Machine Learning/Networks/network.h index fccefd2c7..621447d6e 100644 --- a/Backend/Machine Learning/Networks/network.h +++ b/Backend/Machine Learning/Networks/network.h @@ -58,8 +58,12 @@ class MetaNetwork; class NNetwork { private: + friend RNN; friend MetaNetwork; + DataInput* di; + NNInfo* skeleton; + LayerBuilder meat; CMatrix confusionMatrix; GNet::GServer* serverInstance; GNet::Connection* cConnection; @@ -82,6 +86,7 @@ class NNetwork bool firstRunActivation; // for tables & graphs + shmea::GList learningCurve; std::vector rocCurve; shmea::GList results; shmea::GTable nbRecord; @@ -102,9 +107,9 @@ class NNetwork static const int RUN_TEST = 1; static const int RUN_VALIDATE = 2; - DataInput* di; - NNInfo* skeleton; - LayerBuilder meat; + // DataInput* di; + // NNInfo* skeleton; + // LayerBuilder meat; Terminator terminator; NNetwork(int=TYPE_DFF); @@ -123,7 +128,7 @@ class NNetwork // Stochastic Gradient Descent void train(DataInput*); void test(DataInput*); - + void setInputData(const std::vector>& images, const std::vector& labels); int64_t getID() const; shmea::GString getName() const; NNInfo* getNNInfo(); @@ -131,6 +136,8 @@ class NNetwork // graphing shmea::GList getResults() const; + // const std::vector& getROCCurve() const; + shmea::GList getResults() const; void clean(); void resetGraphs(); }; diff --git a/Backend/Machine Learning/State/LayerBuilder.cpp b/Backend/Machine Learning/State/LayerBuilder.cpp index d26a0f2d8..1555b649e 100644 --- a/Backend/Machine Learning/State/LayerBuilder.cpp +++ b/Backend/Machine Learning/State/LayerBuilder.cpp @@ -548,6 +548,44 @@ void glades::LayerBuilder::clean() xRange = 0.0f; } +void glades::LayerBuilder::build1DConvolutionalLayers(const NNInfo* skeleton) { + int inputLayerSize = inputLayers[0]->size(); + int outputLayerSize = skeleton->getOutputLayerSize(); + int prevLayerSize = inputLayerSize; + int outputType = skeleton->getOutputType(); + bool isPositive = false; + int activationType; + + // Create each 1D convolutional layer + for (int i = 0; i < skeleton->numHiddenLayers(); ++i) { + activationType = skeleton->getActivationType(i); + // Get the current layer size + int cLayerSize = skeleton->getHiddenLayerSize(i); + Layer* cLayer = new Layer(Layer::HIDDEN_TYPE); + + if (isPositive) { + // Create the 1D convolutional layer + cLayer->initWeights(prevLayerSize, cLayerSize, Node::INIT_POSXAVIER, activationType); + } else { + // Check if positive + if ((activationType == GMath::SIGMOID) || (activationType == GMath::RELU) || + (activationType == GMath::LEAKY) || (outputType == GMath::CLASSIFICATION)) { + isPositive = true; + i = -1; + for (unsigned int j = 0; j < layers.size(); ++j) + delete layers[j]; + layers.clear(); + continue; + } + + cLayer->initWeights(prevLayerSize, cLayerSize, Node::INIT_XAVIER, activationType); + } + + layers.push_back(cLayer); + prevLayerSize = cLayerSize; + } +} + // Database /*! * @brief load network diff --git a/Backend/Machine Learning/State/layer.cpp b/Backend/Machine Learning/State/layer.cpp index 8fbda8ae8..15f25e807 100644 --- a/Backend/Machine Learning/State/layer.cpp +++ b/Backend/Machine Learning/State/layer.cpp @@ -279,6 +279,18 @@ void glades::Layer::print() const printf("\n"); } +void glades::Layer::perform1DConvolution(const std::vector& input, std::vector& output, int contextWindowSize) { + // Implement 1D convolution operation + // Ensure the method handles context windows appropriately + for (size_t i = 0; i < input.size() - contextWindowSize + 1; ++i) { + float sum = 0.0f; + for (int j = 0; j < contextWindowSize; ++j) { + sum += input[i + j] * getNode(j)->getWeight(); + } + output[i] = sum; + } +} + Node* glades::Layer::operator[](unsigned int index) { if (index >= size()) @@ -287,13 +299,13 @@ Node* glades::Layer::operator[](unsigned int index) return children[index]; } -void Layer::setupContext() -{ - for(unsigned int i=0;i newNode(new Node()); - newNode->initWeights(1, Node::INIT_POSRAND); - children[i]->setContextNode(newNode); - } -} +// void Layer::setupContext() +// { +// for(unsigned int i=0;i newNode(new Node()); +// newNode->initWeights(1, Node::INIT_POSRAND); +// children[i]->setContextNode(newNode); +// } +// }