[{"data":1,"prerenderedAt":734},["ShallowReactive",2],{"$sTBCGmMSb4":3},{"id":4,"title":5,"body":6,"description":122,"extension":727,"meta":728,"navigation":729,"path":730,"seo":731,"stem":732,"__hash__":733},"content/modules/advanced/neural-network/content.md","Neural Networks",{"type":7,"value":8,"toc":678},"minimark",[9,13,18,22,33,37,42,45,68,72,75,90,94,97,101,104,140,144,156,162,166,169,178,182,188,202,206,212,226,230,236,247,251,254,263,267,293,297,300,306,309,313,319,323,332,336,339,345,348,356,360,363,369,373,377,380,384,404,408,411,422,426,435,439,450,454,465,469,480,484,488,499,503,514,518,535,539,553,557,561,575,579,593,597,611,615,618,623,637,642,656,661],[10,11,5],"h1",{"id":12},"neural-networks",[14,15,17],"h2",{"id":16},"introduction","Introduction",[19,20,21],"p",{},"Neural networks are the foundation of modern deep learning. They're computing systems inspired by biological neural networks that can learn complex patterns from data through interconnected layers of artificial neurons.",[19,23,24,29],{},[25,26],"img",{"alt":27,"src":28},"Neural Network Architecture Overview","https://upload.wikimedia.org/wikipedia/commons/thumb/4/46/Colored_neural_network.svg/500px-Colored_neural_network.svg.png",[30,31,32],"em",{},"A typical feedforward neural network with input, hidden, and output layers",[14,34,36],{"id":35},"network-architecture","Network Architecture",[38,39,41],"h3",{"id":40},"layers","Layers",[19,43,44],{},"A feedforward neural network consists of:",[46,47,48,56,62],"ol",{},[49,50,51,55],"li",{},[52,53,54],"strong",{},"Input Layer",": Receives the raw features",[49,57,58,61],{},[52,59,60],{},"Hidden Layers",": Process information through weighted connections",[49,63,64,67],{},[52,65,66],{},"Output Layer",": Produces the final prediction",[38,69,71],{"id":70},"neurons","Neurons",[19,73,74],{},"Each neuron:",[76,77,78,81,84,87],"ul",{},[49,79,80],{},"Receives inputs from previous layer",[49,82,83],{},"Computes weighted sum plus bias",[49,85,86],{},"Applies activation function",[49,88,89],{},"Passes output to next layer",[14,91,93],{"id":92},"forward-propagation","Forward Propagation",[19,95,96],{},"Forward propagation is how the network makes predictions.",[38,98,100],{"id":99},"step-by-step-process","Step-by-Step Process",[19,102,103],{},"For each neuron in each layer:",[46,105,106,123,134],{},[49,107,108,111,112],{},[52,109,110],{},"Compute weighted sum",":",[113,114,119],"pre",{"className":115,"code":117,"language":118},[116],"language-text","z = w₁x₁ + w₂x₂ + ... + wₙxₙ + b\n","text",[120,121,117],"code",{"__ignoreMap":122},"",[49,124,125,111,128],{},[52,126,127],{},"Apply activation function",[113,129,132],{"className":130,"code":131,"language":118},[116],"a = activation(z)\n",[120,133,131],{"__ignoreMap":122},[49,135,136,139],{},[52,137,138],{},"Pass to next layer",": Output becomes input for next layer",[38,141,143],{"id":142},"example","Example",[19,145,146,147,151,152,155],{},"Input: ",[148,149,150],"span",{},"1.0, 0.5","\nWeights: ",[148,153,154],{},"0.3, 0.7","\nBias: 0.1",[113,157,160],{"className":158,"code":159,"language":118},[116],"z = 0.3(1.0) + 0.7(0.5) + 0.1 = 0.75\na = sigmoid(0.75) = 0.68\n",[120,161,159],{"__ignoreMap":122},[14,163,165],{"id":164},"activation-functions","Activation Functions",[19,167,168],{},"Activation functions introduce non-linearity, allowing networks to learn complex patterns.",[19,170,171,175],{},[25,172],{"alt":173,"src":174},"Common Activation Functions","https://upload.wikimedia.org/wikipedia/commons/thumb/6/6f/Gjl-t%28x%29.svg/500px-Gjl-t%28x%29.svg.png",[30,176,177],{},"Comparison of common activation functions: Sigmoid, Tanh, and ReLU",[38,179,181],{"id":180},"sigmoid","Sigmoid",[113,183,186],{"className":184,"code":185,"language":118},[116],"σ(x) = 1 / (1 + e^(-x))\n",[120,187,185],{"__ignoreMap":122},[76,189,190,193,196,199],{},[49,191,192],{},"Output range: (0, 1)",[49,194,195],{},"Smooth gradient",[49,197,198],{},"Can suffer from vanishing gradients",[49,200,201],{},"Good for output layer in binary classification",[38,203,205],{"id":204},"relu-rectified-linear-unit","ReLU (Rectified Linear Unit)",[113,207,210],{"className":208,"code":209,"language":118},[116],"f(x) = max(0, x)\n",[120,211,209],{"__ignoreMap":122},[76,213,214,217,220,223],{},[49,215,216],{},"Output range: [0, ∞)",[49,218,219],{},"Most popular for hidden layers",[49,221,222],{},"Computationally efficient",[49,224,225],{},"Can suffer from \"dying ReLU\" problem",[38,227,229],{"id":228},"tanh-hyperbolic-tangent","Tanh (Hyperbolic Tangent)",[113,231,234],{"className":232,"code":233,"language":118},[116],"tanh(x) = (e^x - e^(-x)) / (e^x + e^(-x))\n",[120,235,233],{"__ignoreMap":122},[76,237,238,241,244],{},[49,239,240],{},"Output range: (-1, 1)",[49,242,243],{},"Zero-centered (unlike sigmoid)",[49,245,246],{},"Similar gradient issues as sigmoid",[14,248,250],{"id":249},"backpropagation","Backpropagation",[19,252,253],{},"Backpropagation is how networks learn by computing gradients.",[19,255,256,260],{},[25,257],{"alt":258,"src":259},"Backpropagation Process","https://upload.wikimedia.org/wikipedia/commons/thumb/e/e4/Backpropagation.svg/500px-Backpropagation.svg.png",[30,261,262],{},"Forward and backward passes in backpropagation",[38,264,266],{"id":265},"the-algorithm","The Algorithm",[46,268,269,275,281,287],{},[49,270,271,274],{},[52,272,273],{},"Forward pass",": Compute predictions",[49,276,277,280],{},[52,278,279],{},"Compute loss",": Measure error",[49,282,283,286],{},[52,284,285],{},"Backward pass",": Compute gradients",[49,288,289,292],{},[52,290,291],{},"Update weights",": Adjust to reduce loss",[38,294,296],{"id":295},"gradient-computation","Gradient Computation",[19,298,299],{},"For each weight, compute:",[113,301,304],{"className":302,"code":303,"language":118},[116],"∂Loss/∂w = ∂Loss/∂a × ∂a/∂z × ∂z/∂w\n",[120,305,303],{"__ignoreMap":122},[19,307,308],{},"Using the chain rule, gradients flow backwards through the network.",[38,310,312],{"id":311},"weight-update","Weight Update",[113,314,317],{"className":315,"code":316,"language":118},[116],"w_new = w_old - learning_rate × gradient\n",[120,318,316],{"__ignoreMap":122},[14,320,322],{"id":321},"loss-functions","Loss Functions",[19,324,325,329],{},[25,326],{"alt":327,"src":328},"Loss Function Landscape","https://upload.wikimedia.org/wikipedia/commons/thumb/a/a3/Gradient_descent.gif/350px-Gradient_descent.gif",[30,330,331],{},"Gradient descent navigating a loss function landscape",[38,333,335],{"id":334},"binary-cross-entropy","Binary Cross-Entropy",[19,337,338],{},"For binary classification:",[113,340,343],{"className":341,"code":342,"language":118},[116],"Loss = -[y log(ŷ) + (1-y) log(1-ŷ)]\n",[120,344,342],{"__ignoreMap":122},[19,346,347],{},"Where:",[76,349,350,353],{},[49,351,352],{},"y is the true label (0 or 1)",[49,354,355],{},"ŷ is the predicted probability",[38,357,359],{"id":358},"mean-squared-error","Mean Squared Error",[19,361,362],{},"For regression:",[113,364,367],{"className":365,"code":366,"language":118},[116],"Loss = (1/n) Σ(y - ŷ)²\n",[120,368,366],{"__ignoreMap":122},[14,370,372],{"id":371},"training-process","Training Process",[38,374,376],{"id":375},"epoch","Epoch",[19,378,379],{},"One complete pass through the training data.",[38,381,383],{"id":382},"batch-vs-stochastic","Batch vs Stochastic",[76,385,386,392,398],{},[49,387,388,391],{},[52,389,390],{},"Batch Gradient Descent",": Use all data for each update",[49,393,394,397],{},[52,395,396],{},"Stochastic Gradient Descent (SGD)",": Use one sample at a time",[49,399,400,403],{},[52,401,402],{},"Mini-batch",": Use small batches (most common)",[38,405,407],{"id":406},"learning-rate","Learning Rate",[19,409,410],{},"Controls step size for weight updates:",[76,412,413,416,419],{},[49,414,415],{},"Too high: May overshoot minimum",[49,417,418],{},"Too low: Slow convergence",[49,420,421],{},"Typical range: 0.001 - 0.1",[14,423,425],{"id":424},"common-challenges","Common Challenges",[19,427,428,432],{},[25,429],{"alt":430,"src":431},"Vanishing Gradient Problem","https://upload.wikimedia.org/wikipedia/commons/thumb/4/46/Vanishing_gradient_problem.svg/500px-Vanishing_gradient_problem.svg.png",[30,433,434],{},"Gradients diminish as they propagate through deep networks",[38,436,438],{"id":437},"vanishing-gradients","Vanishing Gradients",[76,440,441,444,447],{},[49,442,443],{},"Gradients become very small in deep networks",[49,445,446],{},"Early layers learn slowly",[49,448,449],{},"Solutions: ReLU, batch normalization, residual connections",[38,451,453],{"id":452},"exploding-gradients","Exploding Gradients",[76,455,456,459,462],{},[49,457,458],{},"Gradients become very large",[49,460,461],{},"Weights update too much",[49,463,464],{},"Solutions: Gradient clipping, careful initialization",[38,466,468],{"id":467},"overfitting","Overfitting",[76,470,471,474,477],{},[49,472,473],{},"Network memorizes training data",[49,475,476],{},"Poor generalization to new data",[49,478,479],{},"Solutions: Regularization, dropout, more data",[14,481,483],{"id":482},"best-practices","Best Practices",[38,485,487],{"id":486},"architecture-design","Architecture Design",[76,489,490,493,496],{},[49,491,492],{},"Start simple, add complexity as needed",[49,494,495],{},"More layers = more capacity but harder to train",[49,497,498],{},"Typical hidden layer sizes: 16, 32, 64, 128, 256",[38,500,502],{"id":501},"initialization","Initialization",[76,504,505,508,511],{},[49,506,507],{},"Random initialization breaks symmetry",[49,509,510],{},"Xavier/He initialization scales with layer size",[49,512,513],{},"Never initialize all weights to same value",[38,515,517],{"id":516},"hyperparameter-tuning","Hyperparameter Tuning",[76,519,520,523,526,529,532],{},[49,521,522],{},"Learning rate: Most important",[49,524,525],{},"Number of layers and neurons",[49,527,528],{},"Activation functions",[49,530,531],{},"Batch size",[49,533,534],{},"Number of epochs",[38,536,538],{"id":537},"monitoring-training","Monitoring Training",[76,540,541,544,547,550],{},[49,542,543],{},"Plot loss curves",[49,545,546],{},"Check for overfitting (train vs validation)",[49,548,549],{},"Monitor gradient magnitudes",[49,551,552],{},"Visualize activations",[14,554,556],{"id":555},"applications","Applications",[38,558,560],{"id":559},"computer-vision","Computer Vision",[76,562,563,566,569,572],{},[49,564,565],{},"Image classification",[49,567,568],{},"Object detection",[49,570,571],{},"Facial recognition",[49,573,574],{},"Style transfer",[38,576,578],{"id":577},"natural-language-processing","Natural Language Processing",[76,580,581,584,587,590],{},[49,582,583],{},"Text classification",[49,585,586],{},"Machine translation",[49,588,589],{},"Sentiment analysis",[49,591,592],{},"Question answering",[38,594,596],{"id":595},"other-domains","Other Domains",[76,598,599,602,605,608],{},[49,600,601],{},"Speech recognition",[49,603,604],{},"Game playing (AlphaGo)",[49,606,607],{},"Recommendation systems",[49,609,610],{},"Time series forecasting",[14,612,614],{"id":613},"summary","Summary",[19,616,617],{},"Neural networks are powerful function approximators that:",[19,619,620],{},[52,621,622],{},"Strengths:",[76,624,625,628,631,634],{},[49,626,627],{},"Learn complex non-linear patterns",[49,629,630],{},"Automatic feature learning",[49,632,633],{},"Scalable to large datasets",[49,635,636],{},"State-of-the-art performance on many tasks",[19,638,639],{},[52,640,641],{},"Challenges:",[76,643,644,647,650,653],{},[49,645,646],{},"Require lots of data",[49,648,649],{},"Computationally expensive",[49,651,652],{},"Many hyperparameters to tune",[49,654,655],{},"Can be difficult to interpret",[19,657,658],{},[52,659,660],{},"Key Concepts:",[76,662,663,666,669,672,675],{},[49,664,665],{},"Forward propagation computes predictions",[49,667,668],{},"Backpropagation computes gradients",[49,670,671],{},"Activation functions add non-linearity",[49,673,674],{},"Gradient descent updates weights",[49,676,677],{},"Architecture and hyperparameters matter",{"title":122,"searchDepth":679,"depth":680,"links":681},2,3,[682,683,687,691,696,701,705,710,715,721,726],{"id":16,"depth":679,"text":17},{"id":35,"depth":679,"text":36,"children":684},[685,686],{"id":40,"depth":680,"text":41},{"id":70,"depth":680,"text":71},{"id":92,"depth":679,"text":93,"children":688},[689,690],{"id":99,"depth":680,"text":100},{"id":142,"depth":680,"text":143},{"id":164,"depth":679,"text":165,"children":692},[693,694,695],{"id":180,"depth":680,"text":181},{"id":204,"depth":680,"text":205},{"id":228,"depth":680,"text":229},{"id":249,"depth":679,"text":250,"children":697},[698,699,700],{"id":265,"depth":680,"text":266},{"id":295,"depth":680,"text":296},{"id":311,"depth":680,"text":312},{"id":321,"depth":679,"text":322,"children":702},[703,704],{"id":334,"depth":680,"text":335},{"id":358,"depth":680,"text":359},{"id":371,"depth":679,"text":372,"children":706},[707,708,709],{"id":375,"depth":680,"text":376},{"id":382,"depth":680,"text":383},{"id":406,"depth":680,"text":407},{"id":424,"depth":679,"text":425,"children":711},[712,713,714],{"id":437,"depth":680,"text":438},{"id":452,"depth":680,"text":453},{"id":467,"depth":680,"text":468},{"id":482,"depth":679,"text":483,"children":716},[717,718,719,720],{"id":486,"depth":680,"text":487},{"id":501,"depth":680,"text":502},{"id":516,"depth":680,"text":517},{"id":537,"depth":680,"text":538},{"id":555,"depth":679,"text":556,"children":722},[723,724,725],{"id":559,"depth":680,"text":560},{"id":577,"depth":680,"text":578},{"id":595,"depth":680,"text":596},{"id":613,"depth":679,"text":614},"md",{},true,"/modules/advanced/neural-network/content",{"title":5,"description":122},"modules/advanced/neural-network/content","cAMpyfQqqPtUb-7NN3EoSctCYMZCW2fGuv68ZUANX-w",1792223550839]