ESP32-S3 plant doctor: camera + Gemini + soil sensor

// ESP32-S3 plant doctor: camera + Gemini vision + soil sensor, on a phone dashboard
// Board: XIAO ESP32S3 Sense, with Tools > PSRAM set to "OPI PSRAM"
// Soil sensor (Robot Maker Toolkit, capacitive v1.2): VCC -> 3V3, GND -> GND, AOUT -> D0
// Library: ArduinoJson 7 (Library Manager). WiFi + API key go in secrets.h
#include <WiFi.h>
#include <WebServer.h>
#include <WiFiClientSecure.h>
#include <HTTPClient.h>
#include <ArduinoJson.h>
#include "esp_camera.h"
#include "mbedtls/base64.h"
#include "secrets.h"   // WIFI_NAME, WIFI_PASS, GEMINI_KEY: never commit it

const int SOIL_PIN = A0;      // D0 = GPIO1, ADC1: keeps working with WiFi on
const int DRY_RAW = 2900;     // sensor in air   (measure yours)
const int WET_RAW = 1250;     // sensor in water (measure yours)
const int THIRSTY = 30;       // below this %, water it
const unsigned long PHOTO_EVERY = 30UL * 60 * 1000;   // every 30 min = 48 photos a day

const char* URL = "https://generativelanguage.googleapis.com/v1beta/"
                  "models/gemini-3.5-flash-lite:generateContent";

// The request = HEAD + the photo in base64 + TAIL
const char HEAD[] = R"json({"contents":[{"parts":[
{"text":"You are a houseplant doctor. Look at this photo of a potted plant and report only what you can see. If something is unclear, say so and lower the confidence."},
{"inline_data":{"mime_type":"image/jpeg","data":")json";
const char TAIL[] = R"json("}}]}],
"generationConfig":{"responseMimeType":"application/json","responseSchema":{
"type":"OBJECT","properties":{
"health":{"type":"STRING","enum":["healthy","stressed","sick"]},
"pests":{"type":"STRING","description":"visible pests or disease, or none"},
"leaf_colour":{"type":"STRING","description":"a few words"},
"needs_water":{"type":"BOOLEAN","description":"true if the leaves look thirsty: drooping, curling or dry"},
"confidence":{"type":"NUMBER","description":"0 to 1"},
"advice":{"type":"STRING","description":"one short sentence"}},
"required":["health","pests","leaf_colour","needs_water","confidence","advice"]}}})json";

// Google's root certificate (GTS Root R4, valid until 2036).
// Pin the root, not the server's own certificate: that one changes every few months.
const char GOOGLE_ROOT[] = R"pem(-----BEGIN CERTIFICATE-----
MIICCTCCAY6gAwIBAgINAgPlwGjvYxqccpBQUjAKBggqhkjOPQQDAzBHMQswCQYD
VQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEUMBIG
A1UEAxMLR1RTIFJvb3QgUjQwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAwMDAw
WjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2Vz
IExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjQwdjAQBgcqhkjOPQIBBgUrgQQAIgNi
AATzdHOnaItgrkO4NcWBMHtLSZ37wWHO5t5GvWvVYRg1rkDdc/eJkTBa6zzuhXyi
QHY7qca4R9gq55KRanPpsXI5nymfopjTX15YhmUPoYRlBtHci8nHc8iMai/lxKvR
HYqjQjBAMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW
BBSATNbrdP9JNqPV2Py1PsVq8JQdjDAKBggqhkjOPQQDAwNpADBmAjEA6ED/g94D
9J+uHXqnLrmvT/aDHQ4thQEd0dlq7A/Cr8deVl5c1RxYIigL9zC2L7F8AjEA8GE8
p/SgguMh1YQdc4acLa/KNJvxn7kjNuK8YAOdgLOaVsjh4rsUecrNIdSUtUlD
-----END CERTIFICATE-----
)pem";

WebServer server(80);
int raw = 0;
int moisture = 0;
uint8_t* photo = nullptr;     // the last JPEG, kept in PSRAM for the dashboard
size_t photoLen = 0;
int photoCount = 0;
JsonDocument verdict;         // Gemini's answer
bool firstPhoto = true;
bool wantCheck = false;
unsigned long lastRead = 0;
unsigned long lastPhoto = 0;

bool startCamera() {
  camera_config_t c = {};
  c.pin_pwdn = -1;  c.pin_reset = -1;  c.pin_xclk = 10;
  c.pin_sccb_sda = 40;  c.pin_sccb_scl = 39;
  c.pin_d7 = 48;  c.pin_d6 = 11;  c.pin_d5 = 12;  c.pin_d4 = 14;
  c.pin_d3 = 16;  c.pin_d2 = 18;  c.pin_d1 = 17;  c.pin_d0 = 15;
  c.pin_vsync = 38;  c.pin_href = 47;  c.pin_pclk = 13;
  c.xclk_freq_hz = 20000000;
  c.ledc_timer = LEDC_TIMER_0;
  c.ledc_channel = LEDC_CHANNEL_0;
  c.pixel_format = PIXFORMAT_JPEG;
  c.frame_size = FRAMESIZE_SVGA;          // 800 x 600
  c.jpeg_quality = 12;                    // lower = better + bigger
  c.fb_count = 1;
  c.fb_location = CAMERA_FB_IN_PSRAM;
  return esp_camera_init(&c) == ESP_OK;
}

bool takePhoto() {
  // The first frame can be an old one, and exposure + white balance
  // need a moment, so throw two away and keep the third.
  for (int i = 0; i < 3; i++) {
    camera_fb_t* fb = esp_camera_fb_get();
    if (!fb) return false;
    if (i == 2) {
      free(photo);
      photo = (uint8_t*)ps_malloc(fb->len);
      photoLen = photo ? fb->len : 0;
      if (photo) memcpy(photo, fb->buf, fb->len);
    }
    esp_camera_fb_return(fb);
    delay(300);
  }
  return photo != nullptr;
}

bool askGemini() {
  size_t b64Len = 4 * ((photoLen + 2) / 3);
  size_t headLen = strlen(HEAD), tailLen = strlen(TAIL);
  size_t total = headLen + b64Len + tailLen;
  uint8_t* body = (uint8_t*)ps_malloc(total + 1);   // ~1.4x the photo: PSRAM
  if (!body) return false;
  size_t written = 0;
  memcpy(body, HEAD, headLen);
  mbedtls_base64_encode(body + headLen, b64Len + 1, &written, photo, photoLen);
  memcpy(body + headLen + written, TAIL, tailLen);

  WiFiClientSecure tls;
  tls.setCACert(GOOGLE_ROOT);
  HTTPClient http;
  http.useHTTP10(true);       // a plain reply ArduinoJson can read as a stream
  http.setTimeout(30000);
  http.begin(tls, URL);
  http.addHeader("Content-Type", "application/json");
  http.addHeader("x-goog-api-key", GEMINI_KEY);
  int code = http.POST(body, total);
  free(body);

  bool ok = false;
  if (code == 200) {
    JsonDocument filter;      // keep only the answer's text
    filter["candidates"][0]["content"]["parts"][0]["text"] = true;
    JsonDocument reply;
    deserializeJson(reply, http.getStream(), DeserializationOption::Filter(filter));
    const char* text = reply["candidates"][0]["content"]["parts"][0]["text"];
    ok = text && !deserializeJson(verdict, text);
  } else {
    Serial.printf("Gemini HTTP %d\n", code);   // 429 = rate limit
    Serial.println(http.getString());
  }
  http.end();
  return ok;
}

int readRaw() {
  long sum = 0;
  for (int i = 0; i < 16; i++) {
    sum += analogRead(SOIL_PIN);
    delay(2);
  }
  return sum / 16;
}

int toPercent(int r) {
  return constrain(map(r, DRY_RAW, WET_RAW, 0, 100), 0, 100);
}

String decide() {
  if (verdict.isNull()) return "Waiting for the first photo";
  if (moisture < THIRSTY) return "Water me!";
  if (verdict["needs_water"].as<bool>()) return "Looks thirsty, but the soil isn't dry. Don't water: check light and roots.";
  if (verdict["health"] != "healthy") return verdict["advice"].as<String>();
  return "All good";
}

const char PAGE[] = R"html(
<!doctype html><html><head><meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>Plant doctor</title><style>
body{background:#111;color:#eee;font-family:sans-serif;text-align:center;margin:0;padding:12px}
img{width:100%;max-width:480px;border-radius:12px}
#verdict{font-size:24px;margin:12px 0}
table{margin:auto;text-align:left;font-size:18px}
td{padding:4px 10px}
button{font-size:18px;padding:10px 20px;border-radius:10px}
</style></head><body>
<img id="pic" src="/photo.jpg">
<div id="verdict">--</div>
<table>
<tr><td>Soil</td><td id="soil">--</td></tr>
<tr><td>Health</td><td id="health"></td></tr>
<tr><td>Pests</td><td id="pests"></td></tr>
<tr><td>Leaves</td><td id="leaf"></td></tr>
<tr><td>Confidence</td><td id="conf"></td></tr>
<tr><td>Photo</td><td id="age"></td></tr>
</table>
<p id="advice"></p>
<button onclick="fetch('/check')">Check now</button>
<script>
let shown = -1;
async function tick() {
  try {
    const d = await (await fetch('/data')).json();
    soil.textContent = d.moisture + ' %';
    verdict.textContent = d.verdict;
    age.textContent = Math.round(d.age / 60) + ' min ago';
    if (d.ai) {
      health.textContent = d.ai.health;
      pests.textContent = d.ai.pests;
      leaf.textContent = d.ai.leaf_colour;
      conf.textContent = Math.round(d.ai.confidence * 100) + ' %';
      advice.textContent = d.ai.advice;
    }
    if (d.photo != shown) { shown = d.photo; pic.src = '/photo.jpg?n=' + shown; }
  } catch (e) { verdict.textContent = 'offline'; }
}
tick(); setInterval(tick, 5000);
</script></body></html>
)html";

void handleRoot() {
  server.send(200, "text/html", PAGE);
}

void handleData() {
  JsonDocument d;
  d["moisture"] = moisture;
  d["raw"] = raw;
  d["photo"] = photoCount;    // changes when there's a new photo
  d["age"] = (millis() - lastPhoto) / 1000;
  d["ai"] = verdict;
  d["verdict"] = decide();
  String json;
  serializeJson(d, json);
  server.send(200, "application/json", json);
}

void handlePhoto() {
  if (!photo) {
    server.send(404, "text/plain", "no photo yet");
    return;
  }
  server.send_P(200, "image/jpeg", (const char*)photo, photoLen);
}

void handleCheck() {
  bool tooSoon = !firstPhoto && millis() - lastPhoto < 60000;   // max 1 a minute
  wantCheck = !tooSoon;
  server.send(tooSoon ? 429 : 200, "text/plain", tooSoon ? "wait" : "ok");
}

void setup() {
  Serial.begin(115200);
  delay(1000);
  if (!psramFound()) {
    Serial.println("No PSRAM: set Tools > PSRAM to OPI PSRAM");
    while (true) delay(1000);
  }
  if (!startCamera()) {
    Serial.println("Camera failed: is the camera board seated?");
    while (true) delay(1000);
  }
  analogReadResolution(12);
  WiFi.mode(WIFI_STA);
  WiFi.begin(WIFI_NAME, WIFI_PASS);
  Serial.print("Connecting");
  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }
  Serial.println();
  Serial.print("Open http://");
  Serial.println(WiFi.localIP());
  server.on("/", handleRoot);
  server.on("/data", handleData);
  server.on("/photo.jpg", handlePhoto);
  server.on("/check", handleCheck);
  server.begin();
}

void loop() {
  server.handleClient();
  if (millis() - lastRead >= 2000) {
    lastRead = millis();
    raw = readRaw();
    moisture = toPercent(raw);
  }
  if (firstPhoto || wantCheck || millis() - lastPhoto >= PHOTO_EVERY) {
    firstPhoto = wantCheck = false;
    lastPhoto = millis();
    if (takePhoto()) {
      photoCount++;
      Serial.printf("photo %u bytes, ", photoLen);
      bool ok = askGemini();
      Serial.println(ok ? "Gemini OK" : "Gemini failed");
      if (ok) serializeJson(verdict, Serial);
      Serial.printf("\nheap %u, PSRAM %u free\n", ESP.getFreeHeap(), ESP.getFreePsram());
    }
  }
}

// Your WiFi and Gemini API key (aistudio.google.com/apikey).
// Once a real key is in here, add secrets.h to .gitignore: never push it to a public repo.
#pragma once
const char* WIFI_NAME = "your-wifi";
const char* WIFI_PASS = "your-password";
const char* GEMINI_KEY = "your-gemini-api-key";