migrations/20210925171533_bot_preview.js

@@ -1,10 +0,0 @@
-
-exports.up = function(knex) {
-    return knex.schema.table('urls', function (table) {
-        table.boolean('no_preview').defaultTo(false);
-    });
-};
-
-exports.down = function(knex) {
-  
-};

package.json

@@ -31,7 +31,6 @@
     "fastify-basic-auth": "^2.1.0",
     "fastify-cors": "^6.0.2",
     "fastify-jwt": "^3.0.1",
-    "isbot": "^3.3.3",
     "knex": "^0.21.21",
     "sqlite3": "^5.0.2",
     "uniqid": "^5.3.0"

src/server.js

@@ -2,7 +2,6 @@ const fastify = require('fastify')({ logger: true })
 var uniqid = require('uniqid');
 require('dotenv').config();
 const argon2 = require('argon2');
-const isBot = require('isbot')
 
 let config = {
     domain: process.env.DOMAIN || "localhost:3000",
@@ -75,23 +74,27 @@ fastify.decorate('verifyJWT', function async(request, reply, done) {
 
 //Automagic Amazn redirects on /a/
 fastify.get('/a/:id', async (req, res) => {
+    res.header("X-Robots-Tag","noindex, nofollow");
     res.redirect(302, `https://amazon.de/dp/${req.params.id}`)
     await knex('visits').insert({ shortcode: req.params.id, provider: 'a' });
 })
 
 //Automagic Youtube redirects on /yt/
 fastify.get('/yt/:id', async (req, res) => {
+    res.header("X-Robots-Tag","noindex, nofollow");
     res.redirect(302, `https://youtu.be/${req.params.id}`)
     await knex('visits').insert({ shortcode: req.params.id, provider: 'yt' });
 })
 //Automagic Youtube Playlist redirects on /ytpl/
 fastify.get('/ytpl/:id', async (req, res) => {
+    res.header("X-Robots-Tag","noindex, nofollow");
     res.redirect(302, `https://youtube.com/playlist?list=${req.params.id}`)
     await knex('visits').insert({ shortcode: req.params.id, provider: 'ytpl' });
 })
 
 //Automagic ebay item redirects on /e/
 fastify.get('/e/:id', async (req, res) => {
+    res.header("X-Robots-Tag","noindex, nofollow");
     res.redirect(302, `https://ebay.de/itm/${req.params.id}`)
     await knex('visits').insert({ shortcode: req.params.id, provider: 'e' });
 })
@@ -99,23 +102,19 @@ fastify.get('/e/:id', async (req, res) => {
 //Normal shorturls
 fastify.get('/:shortcode', async (req, res) => {
     const shortcode = req.params.shortcode;
+    console.log(req.headers['user-agent'])
 
     //This should never happen but better safe than 500
     if (!shortcode) {
         return 404;
     }
-    const target = await knex.select('target', 'no_preview')
+    const target = await knex.select('target')
         .from('urls')
         .where('shortcode', '=', shortcode)
         .limit(1);
     if (!target[0]) {
         return 404
     }
-
-    if(isBot(req.headers['user-agent']) && target[0].no_preview){
-        return "Bad Bot!"
-    }
-
     res.redirect(302, target[0].target);
     await knex('visits').insert({ shortcode, provider: 'native' });
 })
@@ -127,7 +126,6 @@ const newUrlSchema = {
         properties: {
             target: { type: 'string' },
             shortcode: { type: 'string' },
-            no_preview: { type: 'boolean' },
         }
     }
 };
@@ -136,7 +134,6 @@ const newUrlSchema = {
 fastify.post('/api', { newUrlSchema }, async (req, res) => {
     const target = req.body?.target;
     let shortcode = req.body?.shortcode;
-    let no_preview = req.body?.no_preview || false;
 
     //Check if the user provided a target
     if (!target) {
@@ -156,7 +153,7 @@ fastify.post('/api', { newUrlSchema }, async (req, res) => {
                 return response;
             }
         }
-        const exists = await knex.select('shortcode', 'no_preview')
+        const exists = await knex.select('shortcode')
             .from('urls')
             .where('target', '=', target)
             .limit(1);
@@ -165,8 +162,7 @@ fastify.post('/api', { newUrlSchema }, async (req, res) => {
             return {
                 url: `${config.getBaseUrl()}/${shortcode}`,
                 shortcode,
-                target,
+                target
-                no_preview: exists[0].no_preview
             }
         }
         shortcode = uniqid();
@@ -188,13 +184,12 @@ fastify.post('/api', { newUrlSchema }, async (req, res) => {
     }
 
     //Create a new db entry
-    await knex('urls').insert({ target, shortcode, no_preview });
+    await knex('urls').insert({ target, shortcode });
 
     return {
         url: `${config.getBaseUrl()}/${shortcode}`,
         shortcode,
-        target,
+        target
-        no_preview
     }
 });
 
@@ -221,7 +216,7 @@ fastify.get('/api/:shortcode', async (req, res) => {
         return 404;
     }
 
-    const exists = await knex.select('shortcode', 'target', 'no_preview')
+    const exists = await knex.select('shortcode', 'target')
         .from('urls')
         .where('shortcode', '=', shortcode)
         .limit(1);
@@ -237,7 +232,6 @@ fastify.get('/api/:shortcode', async (req, res) => {
         url: `${config.getBaseUrl()}/${exists[0].shortcode}`,
         shortcode: exists[0].shortcode,
         target: exists[0].target,
-        no_preview: exists[0].no_preview,
         visits: visits.length
     }
 });
@@ -324,7 +318,7 @@ fastify.after(() => {
 
     //Get all urls api route
     fastify.get('/api', { onRequest: fastify.auth([fastify.basicAuth, fastify.verifyJWT]) }, async (req, res) => {
-        urls = await knex.select('target', 'shortcode', 'no_preview')
+        urls = await knex.select('target', 'shortcode')
             .from('urls');
 
         for (let url of urls) {

yarn.lock

@@ -2228,11 +2228,6 @@ isarray@1.0.0, isarray@~1.0.0:
   resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
   integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=
 
-isbot@^3.3.3:
-  version "3.3.3"
-  resolved "https://registry.yarnpkg.com/isbot/-/isbot-3.3.3.tgz#6a2124a6c1dda5db2d0060c4e292b346ecbd2249"
-  integrity sha512-a3HFPPsvtLroqpuTHHJTaUpPHUO0vjPbptJDzJYkymRvOI8tugWM6zE2oq22w5VOq4A5hrX+YRS7VdIPAgWLfw==
-
 isexe@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10"