Having Problems grabbing an object

I've tried other solutions, but they did not work, so i tried to use raycasts, and they kindda worked out, the problem im having right now is tht when my character goes from left to right (from 1 to -1 for example) the raycast wont hit the object anymore and will drop the object instantaneously (the game i'm making is a 3d adventure game so there is a empty game object right in front of my character and the object that is pickable is parented to that empty game object, but it will fall if I move from right to left or 1 to -1 if you prefer)

Here is the script for the grabbing of objects:

using System.Collections; using System.Collections.Generic; using UnityEngine;

public class GrabFunction : MonoBehaviour {

 public Transform HoldPoint;
 public GameObject grabbedObject;
 public Rigidbody rbGrabbedObject;
 bool grabbed;
 RaycastHit grabHit;
 public float grabDistance = 10f;

 // Use this for initialization
 void Start () {
     rbGrabbedObject = GetComponent<Rigidbody>();
 }
 
 // Update is called once per frame
 void Update () {


     if (Input.GetButton("Fire2"))
     {
         if (!grabbed)
         {
             Vector3 fwd = transform.TransformDirection(Vector3.forward);
             Ray ray = new Ray(transform.position, fwd);
             if (Physics.Raycast(ray, out grabHit))
             {
                 Debug.DrawLine(ray.origin, grabHit.point, Color.red);
                 Debug.Log("hitting");
                 InteractiveObject io = grabHit.collider.GetComponent<InteractiveObject>();
                 if (io != null)
                 {
                     grabbed = true;
                     grabbedObject = io.gameObject;
                     Debug.Log(grabbedObject.name);
                     grabbedObject.GetComponent<Rigidbody>().isKinematic = true;
                     grabbedObject.transform.position = HoldPoint.position;
                     grabbedObject.transform.parent = HoldPoint.transform;
                 }
             }
             else
             {
                 Debug.DrawLine(ray.origin, ray.origin + ray.direction * grabDistance, Color.green);

             }
         }
         if (grabbed)
         {
             grabbed = false;
             grabbedObject.GetComponent<Rigidbody>().isKinematic = false;
             grabbedObject.transform.position = HoldPoint.position;
             grabbedObject.transform.parent = null;
         }
     }
 }

}