Create a custom selection extension by extending ArcObjects
SelectionToolPalette.cs
// Copyright 2012 ESRI
// 
// All rights reserved under the copyright laws of the United States
// and applicable international laws, treaties, and conventions.
// 
// You may freely redistribute and use this sample code, with or
// without modification, provided you include the original copyright
// notice and use restrictions.
// 
// See the use restrictions.
// 

using System;
using System.Drawing;
using System.Runtime.InteropServices;
using ESRI.ArcGIS.ADF.BaseClasses;
using ESRI.ArcGIS.ADF.CATIDs;
using ESRI.ArcGIS.Framework;
using ESRI.ArcGIS.ArcMapUI;
using ESRI.ArcGIS.SystemUI;

namespace SelectionCOMSample
{
  /// <summary>
  /// Summary description for SelectionToolPalette.
  /// </summary>
  [Guid("23a0177f-f011-434a-b7f3-80d718d93fd0")]
  [ClassInterface(ClassInterfaceType.None)]
  [ProgId("SelectionCOMSample.SelectionToolPalette")]
  public sealed class SelectionToolPalette : BaseCommand, IToolPalette
  {
    #region COM Registration Function(s)
    [ComRegisterFunction()]
    [ComVisible(false)]
    static void RegisterFunction(Type registerType)
    {
      // Required for ArcGIS Component Category Registrar support
      ArcGISCategoryRegistration(registerType);

      //
      // TODO: Add any COM registration code here
      //
    }

    [ComUnregisterFunction()]
    [ComVisible(false)]
    static void UnregisterFunction(Type registerType)
    {
      // Required for ArcGIS Component Category Registrar support
      ArcGISCategoryUnregistration(registerType);

      //
      // TODO: Add any COM unregistration code here
      //
    }

    #region ArcGIS Component Category Registrar generated code
    /// <summary>
    /// Required method for ArcGIS Component Category registration -
    /// Do not modify the contents of this method with the code editor.
    /// </summary>
    private static void ArcGISCategoryRegistration(Type registerType)
    {
      string regKey = string.Format("HKEY_CLASSES_ROOT\\CLSID\\{{{0}}}", registerType.GUID);
      MxCommands.Register(regKey);

    }
    /// <summary>
    /// Required method for ArcGIS Component Category unregistration -
    /// Do not modify the contents of this method with the code editor.
    /// </summary>
    private static void ArcGISCategoryUnregistration(Type registerType)
    {
      string regKey = string.Format("HKEY_CLASSES_ROOT\\CLSID\\{{{0}}}", registerType.GUID);
      MxCommands.Unregister(regKey);

    }

    #endregion
    #endregion

    private IApplication m_application;
    public SelectionToolPalette()
    {
      base.m_category = "Developer Samples";
      base.m_caption = "Selection Palette";
      base.m_name = "ESRI_SelectionCOMSample_ToolPalette";  
    }

    #region Overriden Class Methods

    /// <summary>
    /// Occurs when this command is created
    /// </summary>
    /// <param name="hook">Instance of the application</param>
    public override void OnCreate(object hook)
    {
      if (hook == null)
        return;

      m_application = hook as IApplication;

      //Disable if it is not ArcMap
      if (hook is IMxApplication)
        base.m_enabled = true;
      else
        base.m_enabled = false;
    }

    /// <summary>
    /// Occurs when this command is clicked
    /// </summary>
    public override void OnClick()
    {
    }

    #endregion

    #region IToolPalette Members

    public bool MenuStyle
    {
      get { return false; }
    }

    public int PaletteColumns
    {
      get { return 2; }
    }

    public int PaletteItemCount
    {
      get { return 3; }
    }

    public bool TearOff
    {
      get { return false; }
    }

    public string get_PaletteItem(int pos)
    {
      switch (pos)
      {
        case 0:
          return "esriArcMapUI.SelectByPolygonTool";
        case 1:
          return "esriArcMapUI.SelectByLayerCommand";
        case 2:
          return "SelectionCOMSample.SelectByLineTool";
        default:
          return "";
      }
    }

    #endregion
  }
}